Skip to contentSkip to conversation

timer: fix resource leak in finalize

PATCH Superseded Erik Gabriel Carrillo 4 participants 30 messages May 1, 2019 – Jul 5, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 timer: fix resource leak in finalize

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")
---
 lib/librte_timer/rte_timer.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
lib/librte_timer/rte_timer.c +10 -2
index eb46009..fb7a87e 100644
@@ -60,6 +60,7 @@ struct rte_timer_data {
 };
 
 #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
 static struct rte_timer_data *rte_timer_data_arr;
 static const uint32_t default_data_id;
 static uint32_t rte_timer_subsystem_initialized;
@@ -164,6 +165,7 @@ rte_timer_subsystem_init_v1905(void)
 		if (mz == NULL)
 			return -EEXIST;
 
+		rte_timer_data_mz = mz;
 		rte_timer_data_arr = mz->addr;
 
 		rte_timer_data_arr[default_data_id].internal_flags |=
@@ -180,6 +182,7 @@ rte_timer_subsystem_init_v1905(void)
 	if (mz == NULL)
 		return -ENOMEM;
 
+	rte_timer_data_mz = mz;
 	rte_timer_data_arr = mz->addr;
 
 	for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
@@ -205,8 +208,13 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
 void __rte_experimental
 rte_timer_subsystem_finalize(void)
 {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return;
+
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	rte_memzone_free(rte_timer_data_mz);
 
 	rte_timer_subsystem_initialized = 0;
 }
Signature · 1 line
2.6.4
#1 lore raw
Quoted 42 lines · Erik Gabriel Carrillo

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")
---
  lib/librte_timer/rte_timer.c | 12 ++++++++++--
  1 file changed, 10 insertions(+), 2 deletions(-)
lib/librte_timer/rte_timer.c +3 -0
index eb46009..fb7a87e 100644
@@ -60,6 +60,7 @@ struct rte_timer_data {
  };
 
  #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
  static struct rte_timer_data *rte_timer_data_arr;
  static const uint32_t default_data_id;
  static uint32_t rte_timer_subsystem_initialized;
@@ -164,6 +165,7 @@ rte_timer_subsystem_init_v1905(void)
  		if (mz == NULL)
  			return -EEXIST;
 
+		rte_timer_data_mz = mz;
  		rte_timer_data_arr = mz->addr;
 
  		rte_timer_data_arr[default_data_id].internal_flags |=
@@ -180,6 +182,7 @@ rte_timer_subsystem_init_v1905(void)
  	if (mz == NULL)
  		return -ENOMEM;
 
+	rte_timer_data_mz = mz;
  	rte_timer_data_arr = mz->addr;
 
  	for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
@@ -205,8 +208,13 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
  void __rte_experimental
  rte_timer_subsystem_finalize(void)

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217

Quoted 10 lines
lib/librte_timer/rte_timer.c +7 -2
  {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return;
+
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	rte_memzone_free(rte_timer_data_mz);

The patch is a correct fix, but the whole idea of this looks dangerous to me.

If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?

Quoted 4 lines
lib/librte_timer/rte_timer.c +0 -0
  	rte_timer_subsystem_initialized = 0;
  }
 
Signature · 2 lines
Thanks,
Anatoly
#2 lore raw
Quoted 67 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 4:18 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalize

Quoted 27 lines · Erik Gabriel Carrillo

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")
---
  lib/librte_timer/rte_timer.c | 12 ++++++++++--
  1 file changed, 10 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/lib/librte_timer/rte_timer.c index eb46009..fb7a87e 100644

lib/librte_timer/rte_timer.c +1 -0
@@ -60,6 +60,7 @@ struct rte_timer_data {
  };
 
  #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
  static struct rte_timer_data *rte_timer_data_arr;
  static const uint32_t default_data_id;
  static uint32_t rte_timer_subsystem_initialized; @@ -164,6 +165,7 @@

rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return -EEXIST;

lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)

Quoted 4 lines
lib/librte_timer/rte_timer.c +1 -0
+		rte_timer_data_mz = mz;
  		rte_timer_data_arr = mz->addr;
 
  		rte_timer_data_arr[default_data_id].internal_flags |= @@ -

180,6

Quoted 12 lines

+182,7 @@ rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return -ENOMEM;

lib/librte_timer/rte_timer.c +1 -0
+	rte_timer_data_mz = mz;
  	rte_timer_data_arr = mz->addr;
 
  	for (i = 0; i < RTE_MAX_DATA_ELS; i++) { @@ -205,8 +208,13 @@

BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
void __rte_experimental rte_timer_subsystem_finalize(void)
{

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217

Quoted 9 lines
lib/librte_timer/rte_timer.c +7 -2
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return;
+
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	rte_memzone_free(rte_timer_data_mz);

The patch is a correct fix, but the whole idea of this looks dangerous to me.

Quoted 3 lines

If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?

Ah yes - that’s right. Perhaps it would be better to maintain a reference count of some sort such that the last process to exit could cause the memzone_free.

Thanks,
Erik

Quoted 9 lines
Quoted 4 lines
  	rte_timer_subsystem_initialized = 0;
  }

--
Thanks,
Anatoly

#3 lore raw
Quoted 67 lines · Erik G
Quoted 67 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 4:18 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalize

Quoted 27 lines · Erik Gabriel Carrillo

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")
---
   lib/librte_timer/rte_timer.c | 12 ++++++++++--
   1 file changed, 10 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/lib/librte_timer/rte_timer.c index eb46009..fb7a87e 100644

lib/librte_timer/rte_timer.c +1 -0
@@ -60,6 +60,7 @@ struct rte_timer_data {
   };
 
   #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
   static struct rte_timer_data *rte_timer_data_arr;
   static const uint32_t default_data_id;
   static uint32_t rte_timer_subsystem_initialized; @@ -164,6 +165,7 @@

rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return -EEXIST;

lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)

Quoted 4 lines
lib/librte_timer/rte_timer.c +1 -0
+		rte_timer_data_mz = mz;
   		rte_timer_data_arr = mz->addr;
 
   		rte_timer_data_arr[default_data_id].internal_flags |= @@ -

180,6

Quoted 12 lines

+182,7 @@ rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return -ENOMEM;

lib/librte_timer/rte_timer.c +1 -0
+	rte_timer_data_mz = mz;
   	rte_timer_data_arr = mz->addr;
 
   	for (i = 0; i < RTE_MAX_DATA_ELS; i++) { @@ -205,8 +208,13 @@
BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
   void __rte_experimental
   rte_timer_subsystem_finalize(void)
   {

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217

Quoted 9 lines
lib/librte_timer/rte_timer.c +7 -2
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return;
+
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	rte_memzone_free(rte_timer_data_mz);

The patch is a correct fix, but the whole idea of this looks dangerous to me.

Quoted 6 lines
Quoted 3 lines

If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?

Ah yes - that’s right. Perhaps it would be better to maintain a reference count of some sort such that the last process to exit could cause the memzone_free.

It feels like a hack, to be honest. A process can crash or exit without calling rte_eal_cleanup(), which will lead to a memory leak due to refcount being stuck at a value that's not representing reality. It will be saf-er than current approach, but still not ideal.

However, i guess it's a good compromise, if i were to choose between a memory leak and a segfault :D I wonder if there is a better approach.

Quoted 12 lines

Thanks,
Erik

Quoted 9 lines
Quoted 4 lines
   	rte_timer_subsystem_initialized = 0;
   }

--
Thanks,
Anatoly

Signature · 2 lines
Thanks,
Anatoly
#4 lore raw
Quoted 89 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 8:04 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalize

Quoted 51 lines · Erik G
Quoted 51 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 4:18 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalize

Quoted 28 lines · Erik Gabriel Carrillo

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")
---
   lib/librte_timer/rte_timer.c | 12 ++++++++++--
   1 file changed, 10 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/lib/librte_timer/rte_timer.c index eb46009..fb7a87e 100644

lib/librte_timer/rte_timer.c +1 -0
@@ -60,6 +60,7 @@ struct rte_timer_data {
   };
 
   #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
   static struct rte_timer_data *rte_timer_data_arr;
   static const uint32_t default_data_id;
   static uint32_t rte_timer_subsystem_initialized; @@ -164,6 +165,7

@@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return -EEXIST;

lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)

Quoted 4 lines
lib/librte_timer/rte_timer.c +1 -0
+		rte_timer_data_mz = mz;
   		rte_timer_data_arr = mz->addr;
 
   		rte_timer_data_arr[default_data_id].internal_flags |= @@ -

180,6

Quoted 7 lines

+182,7 @@ rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return -ENOMEM;

lib/librte_timer/rte_timer.c +1 -0
+	rte_timer_data_mz = mz;
   	rte_timer_data_arr = mz->addr;
Quoted 16 lines
Quoted 8 lines
Quoted 8 lines
   	for (i = 0; i < RTE_MAX_DATA_ELS; i++) { @@ -205,8 +208,13 @@
BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
   void __rte_experimental
   rte_timer_subsystem_finalize(void)
   {
lib/librte_timer/rte_timer.c +1 -2
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
Quoted 8 lines

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217

Quoted 6 lines
lib/librte_timer/rte_timer.c +6 -0
+		return;
+
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	rte_memzone_free(rte_timer_data_mz);

The patch is a correct fix, but the whole idea of this looks dangerous to

me.

Quoted 6 lines
Quoted 4 lines

If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?

Ah yes - that’s right. Perhaps it would be better to maintain a reference

count of some sort such that the last process to exit could cause the memzone_free.

Quoted 1 line
Quoted 7 lines

It feels like a hack, to be honest. A process can crash or exit without calling rte_eal_cleanup(), which will lead to a memory leak due to refcount being stuck at a value that's not representing reality. It will be saf-er than current approach, but still not ideal.

However, i guess it's a good compromise, if i were to choose between a memory leak and a segfault :D I wonder if there is a better approach.

Ok, I will take a look at that approach then as a first step, since a process can already crash before calling rte_eal_cleanup(), which would result in leaks anyway.

Thanks,
Erik

Quoted 18 lines
Quoted 13 lines

Thanks,
Erik

Quoted 9 lines
Quoted 4 lines
   	rte_timer_subsystem_initialized = 0;
   }

--
Thanks,
Anatoly

--
Thanks,
Anatoly

#5 lore raw

PATCH v2 1/1 timer: fix resource leak in finalize

In reply to Erik Gabriel Carrillo

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")

---
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)

 lib/librte_timer/rte_timer.c | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)
lib/librte_timer/rte_timer.c +17 -3
index eb46009..4771287 100644
@@ -60,6 +60,8 @@ struct rte_timer_data {
 };
 
 #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static rte_atomic16_t *rte_timer_mz_refcnt;
 static struct rte_timer_data *rte_timer_data_arr;
 static const uint32_t default_data_id;
 static uint32_t rte_timer_subsystem_initialized;
@@ -155,6 +157,7 @@ rte_timer_subsystem_init_v1905(void)
 	struct rte_timer_data *data;
 	int i, lcore_id;
 	static const char *mz_name = "rte_timer_mz";
+	size_t data_arr_size = RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
 
 	if (rte_timer_subsystem_initialized)
 		return -EALREADY;
@@ -164,10 +167,14 @@ rte_timer_subsystem_init_v1905(void)
 		if (mz == NULL)
 			return -EEXIST;
 
+		rte_timer_data_mz = mz;
 		rte_timer_data_arr = mz->addr;
+		rte_timer_mz_refcnt =
+				(void *)((char *)mz->addr + data_arr_size);
 
 		rte_timer_data_arr[default_data_id].internal_flags |=
 			FL_ALLOCATED;
+		rte_atomic16_inc(rte_timer_mz_refcnt);
 
 		rte_timer_subsystem_initialized = 1;
 
@@ -175,12 +182,15 @@ rte_timer_subsystem_init_v1905(void)
 	}
 
 	mz = rte_memzone_reserve_aligned(mz_name,
-			RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr),
+			data_arr_size + sizeof(*rte_timer_mz_refcnt),
 			SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
 	if (mz == NULL)
 		return -ENOMEM;
 
+	rte_timer_data_mz = mz;
 	rte_timer_data_arr = mz->addr;
+	rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
+	rte_atomic16_init(rte_timer_mz_refcnt);
 
 	for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
 		data = &rte_timer_data_arr[i];
@@ -193,6 +203,7 @@ rte_timer_subsystem_init_v1905(void)
 	}
 
 	rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
+	rte_atomic16_inc(rte_timer_mz_refcnt);
 
 	rte_timer_subsystem_initialized = 1;
 
@@ -205,8 +216,11 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
 void __rte_experimental
 rte_timer_subsystem_finalize(void)
 {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	if (rte_atomic16_dec_and_test(rte_timer_mz_refcnt))
+		rte_memzone_free(rte_timer_data_mz);
 
 	rte_timer_subsystem_initialized = 0;
 }
Signature · 1 line
2.6.4
#6 lore raw
Quoted 29 lines · Erik Gabriel Carrillo

The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.

Fixes c0749f7096c7 ("timer: allow management in shared memory")

---
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)

  lib/librte_timer/rte_timer.c | 20 +++++++++++++++++---
  1 file changed, 17 insertions(+), 3 deletions(-)
lib/librte_timer/rte_timer.c +2 -0
index eb46009..4771287 100644
@@ -60,6 +60,8 @@ struct rte_timer_data {
  };
 
  #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static rte_atomic16_t *rte_timer_mz_refcnt;
  static struct rte_timer_data *rte_timer_data_arr;
  static const uint32_t default_data_id;
  static uint32_t rte_timer_subsystem_initialized;

lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void) · line 160

Quoted 4 lines
lib/librte_timer/rte_timer.c +1 -0
@@ -155,6 +157,7 @@ rte_timer_subsystem_init_v1905(void)
  	struct rte_timer_data *data;
  	int i, lcore_id;
  	static const char *mz_name = "rte_timer_mz";
+	size_t data_arr_size = RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);

nitpicking, but... const?

Quoted 44 lines
lib/librte_timer/rte_timer.c +9 -1
  	if (rte_timer_subsystem_initialized)
  		return -EALREADY;
@@ -164,10 +167,14 @@ rte_timer_subsystem_init_v1905(void)
  		if (mz == NULL)
  			return -EEXIST;
 
+		rte_timer_data_mz = mz;
  		rte_timer_data_arr = mz->addr;
+		rte_timer_mz_refcnt =
+				(void *)((char *)mz->addr + data_arr_size);
 
  		rte_timer_data_arr[default_data_id].internal_flags |=
  			FL_ALLOCATED;
+		rte_atomic16_inc(rte_timer_mz_refcnt);
 
  		rte_timer_subsystem_initialized = 1;
 
@@ -175,12 +182,15 @@ rte_timer_subsystem_init_v1905(void)
  	}
 
  	mz = rte_memzone_reserve_aligned(mz_name,
-			RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr),
+			data_arr_size + sizeof(*rte_timer_mz_refcnt),
  			SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
  	if (mz == NULL)
  		return -ENOMEM;
 
+	rte_timer_data_mz = mz;
  	rte_timer_data_arr = mz->addr;
+	rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
+	rte_atomic16_init(rte_timer_mz_refcnt);
 
  	for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
  		data = &rte_timer_data_arr[i];
@@ -193,6 +203,7 @@ rte_timer_subsystem_init_v1905(void)
  	}
 
  	rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
+	rte_atomic16_inc(rte_timer_mz_refcnt);
 
  	rte_timer_subsystem_initialized = 1;
 

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223

Quoted 10 lines
lib/librte_timer/rte_timer.c +5 -2
@@ -205,8 +216,11 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
  void __rte_experimental
  rte_timer_subsystem_finalize(void)
  {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	if (rte_atomic16_dec_and_test(rte_timer_mz_refcnt))
+		rte_memzone_free(rte_timer_data_mz);

I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also support a case when secondary initializes after primary stops running.

Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.

I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.

Quoted 4 lines
lib/librte_timer/rte_timer.c +0 -0
  	rte_timer_subsystem_initialized = 0;
  }
 
Signature · 2 lines
Thanks,
Anatoly
#7 lore raw

Hi Anatoly,

Thanks for the review. Comments in-line:

<...snipped...>

Quoted 6 lines
Quoted 6 lines
  #define RTE_MAX_DATA_ELS 64
(quoted diff) +2 -0
+static const struct rte_memzone *rte_timer_data_mz; static
+rte_atomic16_t *rte_timer_mz_refcnt;
  static struct rte_timer_data *rte_timer_data_arr;
  static const uint32_t default_data_id;
  static uint32_t rte_timer_subsystem_initialized; @@ -155,6 +157,7 @@
Quoted 9 lines

Review

Quoted 6 lines

rte_timer_subsystem_init_v1905(void)
struct rte_timer_data *data;
int i, lcore_id;
static const char *mz_name = "rte_timer_mz";

(quoted diff) +2 -0
+	size_t data_arr_size = RTE_MAX_DATA_ELS *
+sizeof(*rte_timer_data_arr);

nitpicking, but... const?

No problem - I'll make this change if this line persists into the next version.

<...snipped...>

Quoted 16 lines

Review

Quoted 2 lines
(quoted diff) +0 -0
@@ -205,8 +216,11 @@

BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223

Quoted 10 lines
lib/librte_timer/rte_timer.c +5 -2
  void __rte_experimental
  rte_timer_subsystem_finalize(void)
  {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	if (rte_atomic16_dec_and_test(rte_timer_mz_refcnt))
+		rte_memzone_free(rte_timer_data_mz);

I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also

Indeed, thanks for catching this.

Quoted 10 lines

support a case when secondary initializes after primary stops running.

Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.

I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.

I don't see a solution here currently either. I'll look at removing the memzone_free() call and possibly the rte_timer_subsystem_finalize() API, since it seems like there's no reason for it to exist if it can't free the allocations.

Regards,
Erik

Quoted 9 lines

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);

Quoted 4 lines
lib/librte_timer/rte_timer.c +0 -0
  	rte_timer_subsystem_initialized = 0;
  }
 

--
Thanks,
Anatoly

#8 lore raw
Quoted 52 lines · Erik G

Hi Anatoly,

Thanks for the review. Comments in-line:

<...snipped...>

Quoted 6 lines
Quoted 6 lines
   #define RTE_MAX_DATA_ELS 64
(quoted diff) +2 -0
+static const struct rte_memzone *rte_timer_data_mz; static
+rte_atomic16_t *rte_timer_mz_refcnt;
   static struct rte_timer_data *rte_timer_data_arr;
   static const uint32_t default_data_id;
   static uint32_t rte_timer_subsystem_initialized; @@ -155,6 +157,7 @@
Quoted 9 lines

Review

Quoted 6 lines

rte_timer_subsystem_init_v1905(void)

   	struct rte_timer_data *data;
   	int i, lcore_id;
   	static const char *mz_name = "rte_timer_mz";
(quoted diff) +2 -0
+	size_t data_arr_size = RTE_MAX_DATA_ELS *
+sizeof(*rte_timer_data_arr);

nitpicking, but... const?

No problem - I'll make this change if this line persists into the next version.

<...snipped...>

Quoted 16 lines

Review

Quoted 2 lines
(quoted diff) +0 -0
@@ -205,8 +216,11 @@

BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223

Quoted 10 lines
lib/librte_timer/rte_timer.c +5 -2
   void __rte_experimental
   rte_timer_subsystem_finalize(void)
   {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	if (rte_atomic16_dec_and_test(rte_timer_mz_refcnt))
+		rte_memzone_free(rte_timer_data_mz);

I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also

Indeed, thanks for catching this.

Quoted 7 lines

support a case when secondary initializes after primary stops running.

Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.

Quoted 7 lines
Quoted 3 lines

I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.

I don't see a solution here currently either. I'll look at removing the memzone_free() call and possibly the rte_timer_subsystem_finalize() API, since it seems like there's no reason for it to exist if it can't free the allocations.

I wonder if there are other places in DPDK where this pattern is used.

Technically, this kind of thing /could/ be resolved by having something in our multiprocess shared memory outside of DPDK heap. I.e. store something in rte_eal_memconfig like some other things do. This change, however, would require an ABI break, so while changing this particular API won't need a deprecation notice, the change itself would.

Quoted 13 lines

Regards,
Erik

Quoted 9 lines

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);

Quoted 4 lines
lib/librte_timer/rte_timer.c +0 -0
   	rte_timer_subsystem_initialized = 0;
   }
 

--
Thanks,
Anatoly

Signature · 2 lines
Thanks,
Anatoly
#9 lore raw
Quoted 74 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Wednesday, May 8, 2019 3:50 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [PATCH v2] timer: fix resource leak in finalize

Quoted 54 lines · Erik G

Hi Anatoly,

Thanks for the review. Comments in-line:

<...snipped...>

Quoted 7 lines
Quoted 7 lines
   #define RTE_MAX_DATA_ELS 64
(quoted diff) +2 -0
+static const struct rte_memzone *rte_timer_data_mz; static
+rte_atomic16_t *rte_timer_mz_refcnt;
   static struct rte_timer_data *rte_timer_data_arr;
   static const uint32_t default_data_id;
   static uint32_t rte_timer_subsystem_initialized; @@ -155,6 +157,7

@@

Quoted 9 lines

Review

Quoted 6 lines

rte_timer_subsystem_init_v1905(void)

   	struct rte_timer_data *data;
   	int i, lcore_id;
   	static const char *mz_name = "rte_timer_mz";
(quoted diff) +2 -0
+	size_t data_arr_size = RTE_MAX_DATA_ELS *
+sizeof(*rte_timer_data_arr);

nitpicking, but... const?

No problem - I'll make this change if this line persists into the next version.

<...snipped...>

Quoted 17 lines

Review

Quoted 2 lines
(quoted diff) +0 -0
@@ -205,8 +216,11 @@

BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223

Quoted 10 lines
lib/librte_timer/rte_timer.c +5 -2
   void __rte_experimental
   rte_timer_subsystem_finalize(void)
   {
-	if (rte_timer_data_arr)
-		rte_free(rte_timer_data_arr);
+	if (!rte_timer_subsystem_initialized)
+		return;
+
+	if (rte_atomic16_dec_and_test(rte_timer_mz_refcnt))
+		rte_memzone_free(rte_timer_data_mz);

I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also

Indeed, thanks for catching this.

Quoted 7 lines

support a case when secondary initializes after primary stops running.

Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.

Quoted 7 lines
Quoted 3 lines

I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.

I don't see a solution here currently either. I'll look at removing the memzone_free() call and possibly the rte_timer_subsystem_finalize() API, since it seems like there's no reason

for it to exist if it can't free the allocations.

Quoted 7 lines

I wonder if there are other places in DPDK where this pattern is used.

Technically, this kind of thing /could/ be resolved by having something in our multiprocess shared memory outside of DPDK heap. I.e. store something in rte_eal_memconfig like some other things do. This change, however, would require an ABI break, so while changing this particular API won't need a deprecation notice, the change itself would.

I like this idea; thanks for the suggestion. I went ahead and submitted a new version of this patch with this approach. It's dependent on one other new patch that allows secondary processes to reserve the memzone. I also submitted a deprecation notice for the ABI break for the change to rte_mem_config.

Thomas, I suspect that I should mark v3 of this patch as "deferred", so I've gone ahead and done that, but let me know if that's wrong.

Thanks,
Erik

Quoted 19 lines
Quoted 14 lines

Regards,
Erik

Quoted 9 lines

lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);

Quoted 4 lines
lib/librte_timer/rte_timer.c +0 -0
   	rte_timer_subsystem_initialized = 0;
   }
 

--
Thanks,
Anatoly

--
Thanks,
Anatoly

#10 lore raw

09/05/2019 01:01, Carrillo, Erik G:

Quoted 7 lines

I like this idea; thanks for the suggestion. I went ahead and submitted a new version of this patch with this approach. It's dependent on one other new patch that allows secondary processes to reserve the memzone. I also submitted a deprecation notice for the ABI break for the change to rte_mem_config.

Thomas, I suspect that I should mark v3 of this patch as "deferred", so I've gone ahead and done that, but let me know if that's wrong.

Any patch targetting 19.08 should be marked as deferred, thanks.

#11 lore raw

PATCH v3 1/1 timer: fix resource leak in finalize

In reply to Erik Gabriel Carrillo

By using a lock added to the rte_mem_config (which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.

---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)

This patch depends on http://patches.dpdk.org/patch/53333/.

changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)

 lib/librte_eal/common/include/rte_eal_memconfig.h |  3 +++
 lib/librte_timer/rte_timer.c                      | 23 ++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)
lib/librte_eal/common/include/rte_eal_memconfig.h +3 -0
index 84aabe3..8cbc09c 100644
@@ -64,6 +64,9 @@ struct rte_mem_config {
 	rte_rwlock_t memory_hotplug_lock;
 	/**< indicates whether memory hotplug request is in progress. */
 
+	rte_spinlock_t timer_subsystem_lock;
+	/**< indicates whether timer subsystem init/finalize is in progress. */
+
 	/* memory segments and zones */
 	struct rte_fbarray memzones; /**< Memzone descriptors. */
 
lib/librte_timer/rte_timer.c +22 -1
index c0f5b87..a055d6e 100644
@@ -26,6 +26,7 @@
 #include <rte_malloc.h>
 #include <rte_compat.h>
 #include <rte_errno.h>
+#include <rte_eal_memconfig.h>
 
 #include "rte_timer.h"
 
@@ -60,7 +61,12 @@ struct rte_timer_data {
 	uint8_t internal_flags;
 };
 
+#define RTE_TIMER_SUBSYSTEM_LOCK \
+	(&rte_eal_get_configuration()->mem_config->timer_subsystem_lock)
+
 #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static int *rte_timer_mz_refcnt;
 static struct rte_timer_data *rte_timer_data_arr;
 static const uint32_t default_data_id;
 static uint32_t rte_timer_subsystem_initialized;
@@ -163,22 +169,27 @@ rte_timer_subsystem_init_v1905(void)
 	if (rte_timer_subsystem_initialized)
 		return -EALREADY;
 
+	rte_spinlock_lock(RTE_TIMER_SUBSYSTEM_LOCK);
 lookup:
 	mz = rte_memzone_lookup(mz_name);
 	if (mz == NULL) {
-		mz = rte_memzone_reserve_aligned(mz_name, data_arr_size,
+		mz = rte_memzone_reserve_aligned(mz_name,
+				data_arr_size + sizeof(*rte_timer_mz_refcnt),
 				SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
 		if (mz == NULL) {
 			if (rte_errno == EEXIST)
 				goto lookup;
 
+			rte_spinlock_unlock(RTE_TIMER_SUBSYSTEM_LOCK);
 			return -ENOMEM;
 		}
 
 		do_full_init = true;
 	}
 
+	rte_timer_data_mz = mz;
 	rte_timer_data_arr = mz->addr;
+	rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
 
 	if (do_full_init) {
 		for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
@@ -195,6 +206,9 @@ rte_timer_subsystem_init_v1905(void)
 	}
 
 	rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
+	(*rte_timer_mz_refcnt)++;
+
+	rte_spinlock_unlock(RTE_TIMER_SUBSYSTEM_LOCK);
 
 	rte_timer_subsystem_initialized = 1;
 
@@ -210,6 +224,13 @@ rte_timer_subsystem_finalize(void)
 	if (!rte_timer_subsystem_initialized)
 		return;
 
+	rte_spinlock_lock(RTE_TIMER_SUBSYSTEM_LOCK);
+
+	if (--(*rte_timer_mz_refcnt) == 0)
+		rte_memzone_free(rte_timer_data_mz);
+
+	rte_spinlock_unlock(RTE_TIMER_SUBSYSTEM_LOCK);
+
 	rte_timer_subsystem_initialized = 0;
 }
 
Signature · 1 line
2.6.4
#12 lore raw
Quoted 27 lines · Erik Gabriel Carrillo

By using a lock added to the rte_mem_config (which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.

---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)

This patch depends on http://patches.dpdk.org/patch/53333/.

changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)

  lib/librte_eal/common/include/rte_eal_memconfig.h |  3 +++
  lib/librte_timer/rte_timer.c                      | 23 ++++++++++++++++++++++-
  2 files changed, 25 insertions(+), 1 deletion(-)
lib/librte_eal/common/include/rte_eal_memconfig.h +0 -0
index 84aabe3..8cbc09c 100644

lib/librte_eal/common/include/rte_eal_memconfig.h struct rte_mem_config { · line 69

Quoted 6 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +3 -0
@@ -64,6 +64,9 @@ struct rte_mem_config {
  	rte_rwlock_t memory_hotplug_lock;
  	/**< indicates whether memory hotplug request is in progress. */
 
+	rte_spinlock_t timer_subsystem_lock;
+	/**< indicates whether timer subsystem init/finalize is in progress. */
+

I believe there's an initialize function somewhere which will initialize all of these locks. This lock should be in there as well.

Other than that, i'm OK with this change, however i feel like /just/ adding this would be a missed opportunity, because next time we want to add something here it will be an ABI break again.

We could perhaps use this opportunity to leave some padding for future data. I'm not sure how would that look like, just an idea floating in my head :)

Signature · 2 lines
Thanks,
Anatoly
#13 lore raw

09/05/2019 10:29, Burakov, Anatoly:

Quoted 38 lines · Erik Gabriel Carrillo
Quoted 27 lines

By using a lock added to the rte_mem_config (which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.

---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)

This patch depends on http://patches.dpdk.org/patch/53333/.

changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)

  lib/librte_eal/common/include/rte_eal_memconfig.h |  3 +++
  lib/librte_timer/rte_timer.c                      | 23 ++++++++++++++++++++++-
  2 files changed, 25 insertions(+), 1 deletion(-)
lib/librte_eal/common/include/rte_eal_memconfig.h +0 -0
index 84aabe3..8cbc09c 100644

lib/librte_eal/common/include/rte_eal_memconfig.h struct rte_mem_config { · line 69

Quoted 6 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +3 -0
@@ -64,6 +64,9 @@ struct rte_mem_config {
  	rte_rwlock_t memory_hotplug_lock;
  	/**< indicates whether memory hotplug request is in progress. */
 
+	rte_spinlock_t timer_subsystem_lock;
+	/**< indicates whether timer subsystem init/finalize is in progress. */
+

I believe there's an initialize function somewhere which will initialize all of these locks. This lock should be in there as well.

Quoted 7 lines

Other than that, i'm OK with this change, however i feel like /just/ adding this would be a missed opportunity, because next time we want to add something here it will be an ABI break again.

We could perhaps use this opportunity to leave some padding for future data. I'm not sure how would that look like, just an idea floating in my head :)

Any update or other opinion?

#14 lore raw
Quoted 43 lines · Thomas Monjalon

09/05/2019 10:29, Burakov, Anatoly:

Quoted 42 lines · Erik Gabriel Carrillo
Quoted 27 lines

By using a lock added to the rte_mem_config (which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.

---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)

This patch depends on http://patches.dpdk.org/patch/53333/.

changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)

   lib/librte_eal/common/include/rte_eal_memconfig.h |  3 +++
   lib/librte_timer/rte_timer.c                      | 23 ++++++++++++++++++++++-
   2 files changed, 25 insertions(+), 1 deletion(-)
lib/librte_eal/common/include/rte_eal_memconfig.h +0 -0
index 84aabe3..8cbc09c 100644

lib/librte_eal/common/include/rte_eal_memconfig.h struct rte_mem_config { · line 69

Quoted 6 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +3 -0
@@ -64,6 +64,9 @@ struct rte_mem_config {
   	rte_rwlock_t memory_hotplug_lock;
   	/**< indicates whether memory hotplug request is in progress. */
 
+	rte_spinlock_t timer_subsystem_lock;
+	/**< indicates whether timer subsystem init/finalize is in progress. */
+

I believe there's an initialize function somewhere which will initialize all of these locks. This lock should be in there as well.

Other than that, i'm OK with this change, however i feel like /just/ adding this would be a missed opportunity, because next time we want to add something here it will be an ABI break again.

Quoted 6 lines
Quoted 3 lines

We could perhaps use this opportunity to leave some padding for future data. I'm not sure how would that look like, just an idea floating in my head :)

Any update or other opinion?

I have a patchset that hides mem config - if we are to add a lock, it would be after adding that patch, as a bugfix (since by then it would n longer be an ABI breakage or an API change).

Signature · 2 lines
Thanks,
Anatoly
#15 lore raw

cover Fix timer resource leak

In reply to Erik Gabriel Carrillo

Previous attempts [1] at fixing the resource leak have been deemed unsuccessful because of limitations around what can be done without breaking the ABI. Now that we've broken the EAL ABI, we can fix this issue properly.

This patchset is adding a new lock API, as well as fixes the actual issue.

The patchset dependson the FreeBSD fixes [2], as well as the mem config patchset [3].

[1] http://patches.dpdk.org/patch/53334/
[2] http://patches.dpdk.org/project/dpdk/list/?series=5161
[3] http://patches.dpdk.org/project/dpdk/list/?series=5162

Anatoly Burakov (2):
eal: add internal locks for timer lib into EAL
timer: fix resource leak in finalize

 lib/librte_eal/common/eal_common_mcfg.c       | 14 +++++++
 lib/librte_eal/common/eal_memcfg.h            |  2 +
 .../common/include/rte_eal_memconfig.h        | 18 ++++++++
 lib/librte_eal/rte_eal_version.map            |  2 +
 lib/librte_timer/rte_timer.c                  | 41 +++++++++++++------
 lib/librte_timer/rte_timer.h                  |  5 ++-
 6 files changed, 67 insertions(+), 15 deletions(-)
Signature · 1 line
2.17.1
#16 lore raw

PATCH 1/2 eal: add internal locks for timer lib into EAL

Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

---
 lib/librte_eal/common/eal_common_mcfg.c        | 14 ++++++++++++++
 lib/librte_eal/common/eal_memcfg.h             |  2 ++
 .../common/include/rte_eal_memconfig.h         | 18 ++++++++++++++++++
 lib/librte_eal/rte_eal_version.map             |  2 ++
 4 files changed, 36 insertions(+)
lib/librte_eal/common/eal_common_mcfg.c +14 -0
index 1825d9083..066549432 100644
@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	rte_rwlock_write_unlock(&mcfg->mplock);
 }
+
+void
+rte_mcfg_timer_lock(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	rte_spinlock_lock(&mcfg->tlock);
+}
+
+void
+rte_mcfg_timer_unlock(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	rte_spinlock_unlock(&mcfg->tlock);
+}
lib/librte_eal/common/eal_memcfg.h +2 -0
index e1aae32df..00370cece 100644
@@ -11,6 +11,7 @@
 #include <rte_memory.h>
 #include <rte_memzone.h>
 #include <rte_pause.h>
+#include <rte_spinlock.h>
 #include <rte_rwlock.h>
 #include <rte_tailq.h>
 
@@ -38,6 +39,7 @@ struct rte_mem_config {
 	rte_rwlock_t mlock;   /**< only used by memzone LIB for thread-safe. */
 	rte_rwlock_t qlock;   /**< used for tailq operation for thread safe. */
 	rte_rwlock_t mplock;  /**< only used by mempool LIB for thread-safe. */
+	rte_spinlock_t tlock; /**< needed for timer lib thread safety. */
 
 	rte_rwlock_t memory_hotplug_lock;
 	/**< indicates whether memory hotplug request is in progress. */
lib/librte_eal/common/include/rte_eal_memconfig.h +18 -0
index 1b615c892..05a32e12a 100644
@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);
 void
 rte_mcfg_mempool_write_unlock(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Lock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
+rte_mcfg_timer_lock(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unlock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
+rte_mcfg_timer_unlock(void);
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/rte_eal_version.map +2 -0
index c28951f65..bc08fc4df 100644
@@ -367,6 +367,8 @@ EXPERIMENTAL {
 	rte_malloc_heap_memory_detach;
 	rte_malloc_heap_memory_remove;
 	rte_malloc_heap_socket_is_external;
+	rte_mcfg_timer_lock;
+	rte_mcfg_timer_unlock;
 	rte_mem_alloc_validator_register;
 	rte_mem_alloc_validator_unregister;
 	rte_mem_check_dma_mask;
Signature · 1 line
2.17.1
author Acked-by on v1 1/2
#17 lore raw

Re: [dpdk-dev] [PATCH 1/2] eal: add internal locks for timer lib into EAL

Hi Anatoly,

Quoted 18 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Tuesday, June 25, 2019 11:12 AM
To: [email protected]
Cc: Carrillo, Erik G <[email protected]>
Subject: [PATCH 1/2] eal: add internal locks for timer lib into EAL

Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

<... snipped ...>

lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 118

Quoted 13 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7 -0
@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);
 void
 rte_mcfg_mempool_write_unlock(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Lock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental

Depending on the decision made for the following thread, the "__rte_experimental" tag location may move:
https://mails.dpdk.org/archives/dev/2019-June/136050.html

Quoted 4 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +4 -0
+rte_mcfg_timer_lock(void);
+
+/**
+ * @warning

lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 132

Quoted 10 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7 -0
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unlock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
+rte_mcfg_timer_unlock(void);
+
 #ifdef __cplusplus
 }
 #endif

<... snipped ...>

Other than that:

#18 lore raw

Re: [dpdk-dev] [PATCH 1/2] eal: add internal locks for timer lib into EAL

In reply to Anatoly Burakov
Quoted 77 lines · Anatoly Burakov

Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

---
 lib/librte_eal/common/eal_common_mcfg.c        | 14 ++++++++++++++
 lib/librte_eal/common/eal_memcfg.h             |  2 ++
 .../common/include/rte_eal_memconfig.h         | 18 ++++++++++++++++++
 lib/librte_eal/rte_eal_version.map             |  2 ++
 4 files changed, 36 insertions(+)
lib/librte_eal/common/eal_common_mcfg.c +5 -0
index 1825d9083..066549432 100644
@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)
        struct rte_mem_config *mcfg =
 rte_eal_get_configuration()->mem_config;
        rte_rwlock_write_unlock(&mcfg->mplock);
 }
+
+void
+rte_mcfg_timer_lock(void)
+{
+       struct rte_mem_config *mcfg =

rte_eal_get_configuration()->mem_config;

lib/librte_eal/common/eal_common_mcfg.c +7 -0
+       rte_spinlock_lock(&mcfg->tlock);
+}
+
+void
+rte_mcfg_timer_unlock(void)
+{
+       struct rte_mem_config *mcfg =

rte_eal_get_configuration()->mem_config;

lib/librte_eal/common/eal_common_mcfg.c +2 -0
+       rte_spinlock_unlock(&mcfg->tlock);
+}
lib/librte_eal/common/eal_memcfg.h +1 -0
index e1aae32df..00370cece 100644
@@ -11,6 +11,7 @@
 #include <rte_memory.h>
 #include <rte_memzone.h>
 #include <rte_pause.h>
+#include <rte_spinlock.h>
 #include <rte_rwlock.h>
 #include <rte_tailq.h>
 
@@ -38,6 +39,7 @@ struct rte_mem_config {
        rte_rwlock_t mlock;   /**< only used by memzone LIB for

thread-safe. */
rte_rwlock_t qlock; /**< used for tailq operation for thread
safe. */
rte_rwlock_t mplock; /**< only used by mempool LIB for
thread-safe. */

lib/librte_eal/common/eal_memcfg.h +1 -0
+       rte_spinlock_t tlock; /**< needed for timer lib thread safety. */
 
 
        rte_rwlock_t memory_hotplug_lock;
        /**< indicates whether memory hotplug request is in progress. */
lib/librte_eal/common/include/rte_eal_memconfig.h +0 -0
index 1b615c892..05a32e12a 100644
@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);
 void

lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 119

Quoted 10 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7 -0
 rte_mcfg_mempool_write_unlock(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Lock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
 

As mentionned by Erik,

rte_experimental
void

+rte_mcfg_timer_lock(void);

lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void);

Quoted 9 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +8 -0
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unlock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
 

rte_experimental
void

Quoted 20 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +2 -0
+rte_mcfg_timer_unlock(void);
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/rte_eal_version.map +2 -1
index c28951f65..bc08fc4df 100644
@@ -367,6 +367,8 @@ EXPERIMENTAL {
        rte_malloc_heap_memory_detach;
        rte_malloc_heap_memory_remove;
        rte_malloc_heap_socket_is_external;
+       rte_mcfg_timer_lock;
+       rte_mcfg_timer_unlock;
        rte_mem_alloc_validator_register;
        rte_mem_alloc_validator_unregister;
        rte_mem_check_dma_mask;
--
Quoted 2 lines

2.17.1

Can you put this at the end of the EXPERIMENTAL block under the 19.08 comment?

Signature · 1 line
David Marchand
#19 lore raw

Re: [dpdk-dev] [PATCH 1/2] eal: add internal locks for timer lib into EAL

Quoted 129 lines · David Marchand
Quoted 77 lines · Anatoly Burakov

Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

---
  lib/librte_eal/common/eal_common_mcfg.c        | 14 ++++++++++++++
  lib/librte_eal/common/eal_memcfg.h             |  2 ++
  .../common/include/rte_eal_memconfig.h         | 18 ++++++++++++++++++
  lib/librte_eal/rte_eal_version.map             |  2 ++
  4 files changed, 36 insertions(+)
lib/librte_eal/common/eal_common_mcfg.c +5 -0
index 1825d9083..066549432 100644
@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)
         struct rte_mem_config *mcfg =
 rte_eal_get_configuration()->mem_config;
         rte_rwlock_write_unlock(&mcfg->mplock);
  }
+
+void
+rte_mcfg_timer_lock(void)
+{
+       struct rte_mem_config *mcfg =

rte_eal_get_configuration()->mem_config;

lib/librte_eal/common/eal_common_mcfg.c +7 -0
+       rte_spinlock_lock(&mcfg->tlock);
+}
+
+void
+rte_mcfg_timer_unlock(void)
+{
+       struct rte_mem_config *mcfg =

rte_eal_get_configuration()->mem_config;

lib/librte_eal/common/eal_common_mcfg.c +2 -0
+       rte_spinlock_unlock(&mcfg->tlock);
+}
lib/librte_eal/common/eal_memcfg.h +1 -0
index e1aae32df..00370cece 100644
@@ -11,6 +11,7 @@
  #include <rte_memory.h>
  #include <rte_memzone.h>
  #include <rte_pause.h>
+#include <rte_spinlock.h>
  #include <rte_rwlock.h>
  #include <rte_tailq.h>
 
@@ -38,6 +39,7 @@ struct rte_mem_config {
         rte_rwlock_t mlock;   /**< only used by memzone LIB for

thread-safe. */
rte_rwlock_t qlock; /**< used for tailq operation for thread
safe. */
rte_rwlock_t mplock; /**< only used by mempool LIB for
thread-safe. */

lib/librte_eal/common/eal_memcfg.h +1 -0
+       rte_spinlock_t tlock; /**< needed for timer lib thread safety. */
 
 
         rte_rwlock_t memory_hotplug_lock;
         /**< indicates whether memory hotplug request is in progress. */
lib/librte_eal/common/include/rte_eal_memconfig.h +0 -0
index 1b615c892..05a32e12a 100644
@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);
  void

lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 119

Quoted 10 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7 -0
  rte_mcfg_mempool_write_unlock(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Lock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
 

As mentionned by Erik,

rte_experimental
void

+rte_mcfg_timer_lock(void);

lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void);

Quoted 9 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +8 -0
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unlock the internal EAL Timer Library lock for exclusive access.
+ */
+void __rte_experimental
 

rte_experimental
void

Quoted 17 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +2 -0
+rte_mcfg_timer_unlock(void);
+
  #ifdef __cplusplus
  }
  #endif
lib/librte_eal/rte_eal_version.map +2 -0
index c28951f65..bc08fc4df 100644
@@ -367,6 +367,8 @@ EXPERIMENTAL {
         rte_malloc_heap_memory_detach;
         rte_malloc_heap_memory_remove;
         rte_malloc_heap_socket_is_external;
+       rte_mcfg_timer_lock;
+       rte_mcfg_timer_unlock;
         rte_mem_alloc_validator_register;
Quoted 9 lines
Quoted 5 lines
lib/librte_eal/rte_eal_version.map +0 -1
         rte_mem_alloc_validator_unregister;
         rte_mem_check_dma_mask;
--

2.17.1

Can you put this at the end of the EXPERIMENTAL block under the 19.08 comment?

OK, will do both for v2.

Signature · 2 lines
Thanks,
Anatoly
#20 lore raw

PATCH 2/2 timer: fix resource leak in finalize

In reply to Anatoly Burakov

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.

However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

---
 lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------
 lib/librte_timer/rte_timer.h |  5 +++--
 2 files changed, 31 insertions(+), 15 deletions(-)
lib/librte_timer/rte_timer.c +28 -13
index dd7953922..08517c120 100644
@@ -13,6 +13,7 @@
 #include <rte_atomic.h>
 #include <rte_common.h>
 #include <rte_cycles.h>
+#include <rte_eal_memconfig.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
 #include <rte_launch.h>
@@ -61,6 +62,8 @@ struct rte_timer_data {
 };
 
 #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static int *volatile rte_timer_mz_refcnt;
 static struct rte_timer_data *rte_timer_data_arr;
 static const uint32_t default_data_id;
 static uint32_t rte_timer_subsystem_initialized;
@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)
 	int i, lcore_id;
 	static const char *mz_name = "rte_timer_mz";
 	const size_t data_arr_size =
-				RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
+			RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
+	const size_t mem_size = data_arr_size + sizeof(*rte_timer_mz_refcnt);
 	bool do_full_init = true;
 
 	if (rte_timer_subsystem_initialized)
 		return -EALREADY;
 
-reserve:
-	rte_errno = 0;
-	mz = rte_memzone_reserve_aligned(mz_name, data_arr_size, SOCKET_ID_ANY,
-					 0, RTE_CACHE_LINE_SIZE);
+	rte_mcfg_timer_lock();
+
+	mz = rte_memzone_lookup(mz_name);
 	if (mz == NULL) {
-		if (rte_errno == EEXIST) {
-			mz = rte_memzone_lookup(mz_name);
-			if (mz == NULL)
-				goto reserve;
-
-			do_full_init = false;
-		} else
+		mz = rte_memzone_reserve_aligned(mz_name, mem_size,
+				SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
+		if (mz == NULL) {
+			rte_mcfg_timer_unlock();
 			return -ENOMEM;
-	}
+		}
+		do_full_init = true;
+	} else
+		do_full_init = false;
 
+	rte_timer_data_mz = mz;
 	rte_timer_data_arr = mz->addr;
+	rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
 
 	if (do_full_init) {
 		for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void)
 	}
 
 	rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
+	(*rte_timer_mz_refcnt)++;
+
+	rte_mcfg_timer_unlock();
 
 	rte_timer_subsystem_initialized = 1;
 
@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)
 	if (!rte_timer_subsystem_initialized)
 		return;
 
+	rte_mcfg_timer_lock();
+
+	if (--(*rte_timer_mz_refcnt) == 0)
+		rte_memzone_free(rte_timer_data_mz);
+
+	rte_mcfg_timer_unlock();
+
 	rte_timer_subsystem_initialized = 0;
 }
 
lib/librte_timer/rte_timer.h +3 -2
index 2196934b2..a7af10176 100644
@@ -170,10 +170,11 @@ int __rte_experimental rte_timer_data_dealloc(uint32_t id);
  * Initializes internal variables (list, locks and so on) for the RTE
  * timer library.
  *
+ * @note
+ *   This function must be called in every process before using the library.
+ *
  * @return
  *   - 0: Success
- *   - -EEXIST: Returned in secondary process when primary process has not
- *      yet initialized the timer subsystem
  *   - -ENOMEM: Unable to allocate memory needed to initialize timer
  *      subsystem
  */
Signature · 1 line
2.17.1
author Acked-by on v1 2/2
#21 lore raw
Quoted 19 lines

-----Original Message-----
From: Burakov, Anatoly
Sent: Tuesday, June 25, 2019 11:12 AM
To: [email protected]
Cc: Robert Sanford <[email protected]>; Carrillo, Erik G <[email protected]>
Subject: [PATCH 2/2] timer: fix resource leak in finalize

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.

However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.

Quoted 6 lines

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

#22 lore raw
In reply to Anatoly Burakov
Quoted 130 lines · Anatoly Burakov

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.

However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

---
 lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------
 lib/librte_timer/rte_timer.h |  5 +++--
 2 files changed, 31 insertions(+), 15 deletions(-)
lib/librte_timer/rte_timer.c +18 -13
index dd7953922..08517c120 100644
@@ -13,6 +13,7 @@
 #include <rte_atomic.h>
 #include <rte_common.h>
 #include <rte_cycles.h>
+#include <rte_eal_memconfig.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
 #include <rte_launch.h>
@@ -61,6 +62,8 @@ struct rte_timer_data {
 };
 
 #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static int *volatile rte_timer_mz_refcnt;
 static struct rte_timer_data *rte_timer_data_arr;
 static const uint32_t default_data_id;
 static uint32_t rte_timer_subsystem_initialized;
@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)
        int i, lcore_id;
        static const char *mz_name = "rte_timer_mz";
        const size_t data_arr_size =
-                               RTE_MAX_DATA_ELS *
 sizeof(*rte_timer_data_arr);
+                       RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
+       const size_t mem_size = data_arr_size +
 sizeof(*rte_timer_mz_refcnt);
        bool do_full_init = true;
 
        if (rte_timer_subsystem_initialized)
                return -EALREADY;
 
-reserve:
-       rte_errno = 0;
-       mz = rte_memzone_reserve_aligned(mz_name, data_arr_size,
 SOCKET_ID_ANY,
-                                        0, RTE_CACHE_LINE_SIZE);
+       rte_mcfg_timer_lock();
+
+       mz = rte_memzone_lookup(mz_name);
        if (mz == NULL) {
-               if (rte_errno == EEXIST) {
-                       mz = rte_memzone_lookup(mz_name);
-                       if (mz == NULL)
-                               goto reserve;
-
-                       do_full_init = false;
-               } else
+               mz = rte_memzone_reserve_aligned(mz_name, mem_size,
+                               SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
+               if (mz == NULL) {
+                       rte_mcfg_timer_unlock();
                        return -ENOMEM;
-       }
+               }
+               do_full_init = true;
+       } else
+               do_full_init = false;
 
+       rte_timer_data_mz = mz;
        rte_timer_data_arr = mz->addr;
+       rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
        if (do_full_init) {
                for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
lib/librte_timer/rte_timer.c +10 -0
@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void)
        }
 
        rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
+       (*rte_timer_mz_refcnt)++;
+
+       rte_mcfg_timer_unlock();
 
        rte_timer_subsystem_initialized = 1;
 
@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)
        if (!rte_timer_subsystem_initialized)
                return;
 
+       rte_mcfg_timer_lock();
+
+       if (--(*rte_timer_mz_refcnt) == 0)
+               rte_memzone_free(rte_timer_data_mz);
+
+       rte_mcfg_timer_unlock();
+
        rte_timer_subsystem_initialized = 0;
 }
 
lib/librte_timer/rte_timer.h +2 -0
index 2196934b2..a7af10176 100644
@@ -170,10 +170,11 @@ int __rte_experimental
 rte_timer_data_dealloc(uint32_t id);
  * Initializes internal variables (list, locks and so on) for the RTE
  * timer library.
  *
+ * @note
+ *   This function must be called in every process before using the

library.
+ *
* @return

Quoted 9 lines

* - 0: Success
- * - -EEXIST: Returned in secondary process when primary process has not
- * yet initialized the timer subsystem
* - -ENOMEM: Unable to allocate memory needed to initialize timer
* subsystem
*/
--
2.17.1

Can be squashed with the first patch.

Signature · 1 line
David Marchand
#23 lore raw
Quoted 136 lines · David Marchand
Quoted 133 lines · Anatoly Burakov

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.

However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

---
  lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------
  lib/librte_timer/rte_timer.h |  5 +++--
  2 files changed, 31 insertions(+), 15 deletions(-)
lib/librte_timer/rte_timer.c +18 -13
index dd7953922..08517c120 100644
@@ -13,6 +13,7 @@
  #include <rte_atomic.h>
  #include <rte_common.h>
  #include <rte_cycles.h>
+#include <rte_eal_memconfig.h>
  #include <rte_per_lcore.h>
  #include <rte_memory.h>
  #include <rte_launch.h>
@@ -61,6 +62,8 @@ struct rte_timer_data {
  };
 
  #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static int *volatile rte_timer_mz_refcnt;
  static struct rte_timer_data *rte_timer_data_arr;
  static const uint32_t default_data_id;
  static uint32_t rte_timer_subsystem_initialized;
@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)
         int i, lcore_id;
         static const char *mz_name = "rte_timer_mz";
         const size_t data_arr_size =
-                               RTE_MAX_DATA_ELS *
 sizeof(*rte_timer_data_arr);
+                       RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
+       const size_t mem_size = data_arr_size +
 sizeof(*rte_timer_mz_refcnt);
         bool do_full_init = true;
 
         if (rte_timer_subsystem_initialized)
                 return -EALREADY;
 
-reserve:
-       rte_errno = 0;
-       mz = rte_memzone_reserve_aligned(mz_name, data_arr_size,
 SOCKET_ID_ANY,
-                                        0, RTE_CACHE_LINE_SIZE);
+       rte_mcfg_timer_lock();
+
+       mz = rte_memzone_lookup(mz_name);
         if (mz == NULL) {
-               if (rte_errno == EEXIST) {
-                       mz = rte_memzone_lookup(mz_name);
-                       if (mz == NULL)
-                               goto reserve;
-
-                       do_full_init = false;
-               } else
+               mz = rte_memzone_reserve_aligned(mz_name, mem_size,
+                               SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
+               if (mz == NULL) {
+                       rte_mcfg_timer_unlock();
                         return -ENOMEM;
-       }
+               }
+               do_full_init = true;
+       } else
+               do_full_init = false;
 
+       rte_timer_data_mz = mz;
         rte_timer_data_arr = mz->addr;
+       rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
         if (do_full_init) {
                 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
lib/librte_timer/rte_timer.c +10 -0
@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void)
         }
 
         rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
+       (*rte_timer_mz_refcnt)++;
+
+       rte_mcfg_timer_unlock();
 
         rte_timer_subsystem_initialized = 1;
 
@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)
         if (!rte_timer_subsystem_initialized)
                 return;
 
+       rte_mcfg_timer_lock();
+
+       if (--(*rte_timer_mz_refcnt) == 0)
+               rte_memzone_free(rte_timer_data_mz);
+
+       rte_mcfg_timer_unlock();
+
         rte_timer_subsystem_initialized = 0;
  }
 
lib/librte_timer/rte_timer.h +2 -0
index 2196934b2..a7af10176 100644
@@ -170,10 +170,11 @@ int __rte_experimental
 rte_timer_data_dealloc(uint32_t id);
   * Initializes internal variables (list, locks and so on) for the RTE
   * timer library.
   *
+ * @note
+ *   This function must be called in every process before using the

library.
+ *
* @return
* - 0: Success
- * - -EEXIST: Returned in secondary process when primary process has not
- * yet initialized the timer subsystem

Quoted 9 lines
Quoted 6 lines

* - -ENOMEM: Unable to allocate memory needed to initialize timer
* subsystem
*/
--
2.17.1

Can be squashed with the first patch.

This is not just search-and-replace - this is also a bugfix. I can squash the search-and-replace part with the first patch, but this commit will have to stay IMO.

Signature · 2 lines
Thanks,
Anatoly
#24 lore raw
Quoted 148 lines · Anatoly
Quoted 1 line · David Marchand

On Tue, Jun 25, 2019 at 6:12 PM Anatoly Burakov <

[email protected]>

Quoted 88 lines
Quoted 86 lines · wrote:

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.

However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

---
  lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------
  lib/librte_timer/rte_timer.h |  5 +++--
  2 files changed, 31 insertions(+), 15 deletions(-)
lib/librte_timer/rte_timer.c +16 -13
index dd7953922..08517c120 100644
@@ -13,6 +13,7 @@
  #include <rte_atomic.h>
  #include <rte_common.h>
  #include <rte_cycles.h>
+#include <rte_eal_memconfig.h>
  #include <rte_per_lcore.h>
  #include <rte_memory.h>
  #include <rte_launch.h>
@@ -61,6 +62,8 @@ struct rte_timer_data {
  };
 
  #define RTE_MAX_DATA_ELS 64
+static const struct rte_memzone *rte_timer_data_mz;
+static int *volatile rte_timer_mz_refcnt;
  static struct rte_timer_data *rte_timer_data_arr;
  static const uint32_t default_data_id;
  static uint32_t rte_timer_subsystem_initialized;
@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)
         int i, lcore_id;
         static const char *mz_name = "rte_timer_mz";
         const size_t data_arr_size =
-                               RTE_MAX_DATA_ELS *
 sizeof(*rte_timer_data_arr);
+                       RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
+       const size_t mem_size = data_arr_size +
 sizeof(*rte_timer_mz_refcnt);
         bool do_full_init = true;
 
         if (rte_timer_subsystem_initialized)
                 return -EALREADY;
 
-reserve:
-       rte_errno = 0;
-       mz = rte_memzone_reserve_aligned(mz_name, data_arr_size,
 SOCKET_ID_ANY,
-                                        0, RTE_CACHE_LINE_SIZE);
+       rte_mcfg_timer_lock();
+
+       mz = rte_memzone_lookup(mz_name);
         if (mz == NULL) {
-               if (rte_errno == EEXIST) {
-                       mz = rte_memzone_lookup(mz_name);
-                       if (mz == NULL)
-                               goto reserve;
-
-                       do_full_init = false;
-               } else
+               mz = rte_memzone_reserve_aligned(mz_name, mem_size,
+                               SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
+               if (mz == NULL) {
+                       rte_mcfg_timer_unlock();
                         return -ENOMEM;
-       }
+               }
+               do_full_init = true;
+       } else
+               do_full_init = false;
 
Quoted 3 lines

lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)

Quoted 3 lines
lib/librte_timer/rte_timer.c +2 -0
+       rte_timer_data_mz = mz;
         rte_timer_data_arr = mz->addr;
+       rte_timer_mz_refcnt = (void *)((char *)mz->addr +

data_arr_size);

Quoted 7 lines

lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void) · line 202

Quoted 7 lines
         if (do_full_init) {
                 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
lib/librte_timer/rte_timer.c +0 -0
@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void)
         }
 
         rte_timer_data_arr[default_data_id].internal_flags |=

FL_ALLOCATED;

Quoted 20 lines
Quoted 20 lines
lib/librte_timer/rte_timer.c +10 -0
+       (*rte_timer_mz_refcnt)++;
+
+       rte_mcfg_timer_unlock();
 
         rte_timer_subsystem_initialized = 1;
 
@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)
         if (!rte_timer_subsystem_initialized)
                 return;
 
+       rte_mcfg_timer_lock();
+
+       if (--(*rte_timer_mz_refcnt) == 0)
+               rte_memzone_free(rte_timer_data_mz);
+
+       rte_mcfg_timer_unlock();
+
         rte_timer_subsystem_initialized = 0;
  }
 
Quoted 16 lines
Quoted 11 lines
lib/librte_timer/rte_timer.h +2 -0
index 2196934b2..a7af10176 100644
@@ -170,10 +170,11 @@ int __rte_experimental
 rte_timer_data_dealloc(uint32_t id);
   * Initializes internal variables (list, locks and so on) for the RTE
   * timer library.
   *
+ * @note
+ *   This function must be called in every process before using the
Quoted 5 lines

library.
+ *
* @return
* - 0: Success
- * - -EEXIST: Returned in secondary process when primary process has

not

Quoted 8 lines
Quoted 7 lines

- * yet initialized the timer subsystem
* - -ENOMEM: Unable to allocate memory needed to initialize timer
* subsystem
*/
--
2.17.1

Quoted 7 lines
Quoted 2 lines

Can be squashed with the first patch.

This is not just search-and-replace - this is also a bugfix. I can squash the search-and-replace part with the first patch, but this commit will have to stay IMO.

Yes this is not search and replace, but the first patch is there for the bugfix.
There are no other uses of the newly introduced API, so the whole is a single change to me.

Apart from that, I am ok with the changes, you can add my review tag.

Signature · 1 line
David Marchand
#25 lore raw

v2 cover Fix timer resource leak

In reply to Anatoly Burakov

Previous attempts [1] at fixing the resource leak have been deemed unsuccessful because of limitations around what can be done without breaking the ABI. Now that we've broken the EAL ABI, we can fix this issue properly.

This patchset is adding a new lock API, as well as fixes the actual issue.

The patchset depends on mem config patchset [2].

[1] http://patches.dpdk.org/patch/53334/
[2] http://patches.dpdk.org/project/dpdk/list/?series=5369

Anatoly Burakov (1):
timer: fix resource leak in finalize

 lib/librte_eal/common/eal_common_mcfg.c       | 29 +++++++++++++
 lib/librte_eal/common/eal_memcfg.h            |  8 ++++
 .../common/include/rte_eal_memconfig.h        | 22 ++++++++++
 lib/librte_eal/freebsd/eal/eal.c              |  5 ++-
 lib/librte_eal/linux/eal/eal.c                |  5 ++-
 lib/librte_eal/rte_eal_version.map            |  2 +
 lib/librte_timer/rte_timer.c                  | 41 +++++++++++++------
 lib/librte_timer/rte_timer.h                  |  5 ++-
 8 files changed, 100 insertions(+), 17 deletions(-)
Signature · 1 line
2.17.1
#26 lore raw

PATCH v2 1/1 timer: fix resource leak in finalize

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions [1] because the data itself must live in shared memory.

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races, and implement the synchronization mechanism based on this lock in the timer library.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

Fixes c0749f7096c7 ("timer: allow management in shared memory")

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

---
 lib/librte_eal/common/eal_common_mcfg.c       | 29 +++++++++++++
 lib/librte_eal/common/eal_memcfg.h            |  8 ++++
 .../common/include/rte_eal_memconfig.h        | 22 ++++++++++
 lib/librte_eal/freebsd/eal/eal.c              |  5 ++-
 lib/librte_eal/linux/eal/eal.c                |  5 ++-
 lib/librte_eal/rte_eal_version.map            |  2 +
 lib/librte_timer/rte_timer.c                  | 41 +++++++++++++------
 lib/librte_timer/rte_timer.h                  |  5 ++-
 8 files changed, 100 insertions(+), 17 deletions(-)
lib/librte_eal/common/eal_common_mcfg.c +29 -0
index fe8d2b726..066549432 100644
@@ -4,6 +4,7 @@
 
 #include <rte_config.h>
 #include <rte_eal_memconfig.h>
+#include <rte_version.h>
 
 #include "eal_internal_cfg.h"
 #include "eal_memcfg.h"
@@ -31,6 +32,18 @@ eal_mcfg_wait_complete(void)
 		rte_pause();
 }
 
+int
+eal_mcfg_check_version(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	/* check if version from memconfig matches compiled in macro */
+	if (mcfg->version != RTE_VERSION)
+		return -1;
+
+	return 0;
+}
+
 void
 eal_mcfg_update_internal(void)
 {
@@ -47,6 +60,8 @@ eal_mcfg_update_from_internal(void)
 
 	mcfg->legacy_mem = internal_config.legacy_mem;
 	mcfg->single_file_segments = internal_config.single_file_segments;
+	/* record current DPDK version */
+	mcfg->version = RTE_VERSION;
 }
 
 void
@@ -132,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	rte_rwlock_write_unlock(&mcfg->mplock);
 }
+
+void
+rte_mcfg_timer_lock(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	rte_spinlock_lock(&mcfg->tlock);
+}
+
+void
+rte_mcfg_timer_unlock(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	rte_spinlock_unlock(&mcfg->tlock);
+}
lib/librte_eal/common/eal_memcfg.h +8 -0
index 573233896..359beb216 100644
@@ -11,6 +11,7 @@
 #include <rte_memory.h>
 #include <rte_memzone.h>
 #include <rte_pause.h>
+#include <rte_spinlock.h>
 #include <rte_rwlock.h>
 #include <rte_tailq.h>
 
@@ -19,6 +20,8 @@
  */
 struct rte_mem_config {
 	volatile uint32_t magic;   /**< Magic number - sanity check. */
+	uint32_t version;
+	/**< Prevent secondary processes using different DPDK versions. */
 
 	/* memory topology */
 	uint32_t nchannel;    /**< Number of channels (0 if unknown). */
@@ -34,6 +37,7 @@ struct rte_mem_config {
 	rte_rwlock_t mlock;   /**< used by memzones for thread safety. */
 	rte_rwlock_t qlock;   /**< used by tailqs for thread safety. */
 	rte_rwlock_t mplock;  /**< used by mempool library for thread safety. */
+	rte_spinlock_t tlock; /**< used by timer library for thread safety. */
 
 	rte_rwlock_t memory_hotplug_lock;
 	/**< Indicates whether memory hotplug request is in progress. */
@@ -81,6 +85,10 @@ eal_mcfg_update_from_internal(void);
 void
 eal_mcfg_wait_complete(void);
 
+/* check if DPDK version of current process matches one stored in the config */
+int
+eal_mcfg_check_version(void);
+
 /* set mem config as complete */
 void
 eal_mcfg_complete(void);
lib/librte_eal/common/include/rte_eal_memconfig.h +22 -0
index dc61a6fed..34b0e44a0 100644
@@ -5,6 +5,8 @@
 #ifndef _RTE_EAL_MEMCONFIG_H_
 #define _RTE_EAL_MEMCONFIG_H_
 
+#include <rte_compat.h>
+
 /**
  * @file
  *
@@ -87,6 +89,26 @@ rte_mcfg_mempool_write_lock(void);
 void
 rte_mcfg_mempool_write_unlock(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Lock the internal EAL Timer Library lock for exclusive access.
+ */
+__rte_experimental
+void
+rte_mcfg_timer_lock(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unlock the internal EAL Timer Library lock for exclusive access.
+ */
+__rte_experimental
+void
+rte_mcfg_timer_unlock(void);
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/freebsd/eal/eal.c +4 -1
index ec1650c43..1d3d7e80f 100644
@@ -385,6 +385,10 @@ rte_config_init(void)
 		if (rte_eal_config_attach() < 0)
 			return -1;
 		eal_mcfg_wait_complete();
+		if (eal_mcfg_check_version() < 0) {
+			RTE_LOG(ERR, EAL, "Primary and secondary process DPDK version mismatch\n");
+			return -1;
+		}
 		if (rte_eal_config_reattach() < 0)
 			return -1;
 		eal_mcfg_update_internal();
@@ -395,7 +399,6 @@ rte_config_init(void)
 			rte_config.process_type);
 		return -1;
 	}
-
 	return 0;
 }
 
lib/librte_eal/linux/eal/eal.c +4 -1
index 445d72f0c..a5e0d1e75 100644
@@ -489,6 +489,10 @@ rte_config_init(void)
 		if (rte_eal_config_attach() < 0)
 			return -1;
 		eal_mcfg_wait_complete();
+		if (eal_mcfg_check_version() < 0) {
+			RTE_LOG(ERR, EAL, "Primary and secondary process DPDK version mismatch\n");
+			return -1;
+		}
 		if (rte_eal_config_reattach() < 0)
 			return -1;
 		eal_mcfg_update_internal();
@@ -499,7 +503,6 @@ rte_config_init(void)
 			rte_config.process_type);
 		return -1;
 	}
-
 	return 0;
 }
 
lib/librte_eal/rte_eal_version.map +2 -0
index cc4ef04a0..d0ee67dbf 100644
@@ -405,4 +405,6 @@ EXPERIMENTAL {
 	# added in 19.08
 	rte_lcore_cpuset;
 	rte_lcore_to_cpu_id;
+	rte_mcfg_timer_lock;
+	rte_mcfg_timer_unlock;
 };
lib/librte_timer/rte_timer.c +28 -13

80 lines not on this page · raw message

lib/librte_timer/rte_timer.h +3 -2
index 1b0d5f6a5..05d287d8f 100644
@@ -172,10 +172,11 @@ int rte_timer_data_dealloc(uint32_t id);
  * Initializes internal variables (list, locks and so on) for the RTE
  * timer library.
  *
+ * @note
+ *   This function must be called in every process before using the library.
+ *
  * @return
  *   - 0: Success
- *   - -EEXIST: Returned in secondary process when primary process has not
- *      yet initialized the timer subsystem
  *   - -ENOMEM: Unable to allocate memory needed to initialize timer
  *      subsystem
  */
Signature · 1 line
2.17.1
#27 lore raw

v3 cover Fix timer resource leak

In reply to Anatoly Burakov

Previous attempts [1] at fixing the resource leak have been deemed unsuccessful because of limitations around what can be done without breaking the ABI. Now that we've broken the EAL ABI, we can fix this issue properly.

This patchset is adding a new lock API, as well as fixes the actual issue.

The patchset depends on mem config patchset [2].

v3:
- Accidentally squashed with previous patch, reverted

v2:
- Addressed review comments and squashed patches

[1] http://patches.dpdk.org/patch/53334/
[2] http://patches.dpdk.org/project/dpdk/list/?series=5369

Anatoly Burakov (1):
timer: fix resource leak in finalize

 lib/librte_eal/common/eal_common_mcfg.c       | 14 +++++++
 lib/librte_eal/common/eal_memcfg.h            |  2 +
 .../common/include/rte_eal_memconfig.h        | 22 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  2 +
 lib/librte_timer/rte_timer.c                  | 41 +++++++++++++------
 lib/librte_timer/rte_timer.h                  |  5 ++-
 6 files changed, 71 insertions(+), 15 deletions(-)
Signature · 1 line
2.17.1
#28 lore raw

PATCH v3 1/1 timer: fix resource leak in finalize

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions [1] because the data itself must live in shared memory.

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races, and implement the synchronization mechanism based on this lock in the timer library.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

Fixes c0749f7096c7 ("timer: allow management in shared memory")

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

---
 lib/librte_eal/common/eal_common_mcfg.c       | 14 +++++++
 lib/librte_eal/common/eal_memcfg.h            |  2 +
 .../common/include/rte_eal_memconfig.h        | 22 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  2 +
 lib/librte_timer/rte_timer.c                  | 41 +++++++++++++------
 lib/librte_timer/rte_timer.h                  |  5 ++-
 6 files changed, 71 insertions(+), 15 deletions(-)
lib/librte_eal/common/eal_common_mcfg.c +14 -0
index 1825d9083..066549432 100644
@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	rte_rwlock_write_unlock(&mcfg->mplock);
 }
+
+void
+rte_mcfg_timer_lock(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	rte_spinlock_lock(&mcfg->tlock);
+}
+
+void
+rte_mcfg_timer_unlock(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	rte_spinlock_unlock(&mcfg->tlock);
+}
lib/librte_eal/common/eal_memcfg.h +2 -0
index 030f903ad..359beb216 100644
@@ -11,6 +11,7 @@
 #include <rte_memory.h>
 #include <rte_memzone.h>
 #include <rte_pause.h>
+#include <rte_spinlock.h>
 #include <rte_rwlock.h>
 #include <rte_tailq.h>
 
@@ -36,6 +37,7 @@ struct rte_mem_config {
 	rte_rwlock_t mlock;   /**< used by memzones for thread safety. */
 	rte_rwlock_t qlock;   /**< used by tailqs for thread safety. */
 	rte_rwlock_t mplock;  /**< used by mempool library for thread safety. */
+	rte_spinlock_t tlock; /**< used by timer library for thread safety. */
 
 	rte_rwlock_t memory_hotplug_lock;
 	/**< Indicates whether memory hotplug request is in progress. */
lib/librte_eal/common/include/rte_eal_memconfig.h +22 -0
index dc61a6fed..34b0e44a0 100644
@@ -5,6 +5,8 @@
 #ifndef _RTE_EAL_MEMCONFIG_H_
 #define _RTE_EAL_MEMCONFIG_H_
 
+#include <rte_compat.h>
+
 /**
  * @file
  *
@@ -87,6 +89,26 @@ rte_mcfg_mempool_write_lock(void);
 void
 rte_mcfg_mempool_write_unlock(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Lock the internal EAL Timer Library lock for exclusive access.
+ */
+__rte_experimental
+void
+rte_mcfg_timer_lock(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unlock the internal EAL Timer Library lock for exclusive access.
+ */
+__rte_experimental
+void
+rte_mcfg_timer_unlock(void);
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/rte_eal_version.map +2 -0
index cc4ef04a0..d0ee67dbf 100644
@@ -405,4 +405,6 @@ EXPERIMENTAL {
 	# added in 19.08
 	rte_lcore_cpuset;
 	rte_lcore_to_cpu_id;
+	rte_mcfg_timer_lock;
+	rte_mcfg_timer_unlock;
 };
lib/librte_timer/rte_timer.c +28 -13

80 lines not on this page · raw message

lib/librte_timer/rte_timer.h +3 -2
index 1b0d5f6a5..05d287d8f 100644
@@ -172,10 +172,11 @@ int rte_timer_data_dealloc(uint32_t id);
  * Initializes internal variables (list, locks and so on) for the RTE
  * timer library.
  *
+ * @note
+ *   This function must be called in every process before using the library.
+ *
  * @return
  *   - 0: Success
- *   - -EEXIST: Returned in secondary process when primary process has not
- *      yet initialized the timer subsystem
  *   - -ENOMEM: Unable to allocate memory needed to initialize timer
  *      subsystem
  */
Signature · 1 line
2.17.1
#29 lore raw

05/07/2019 19:22, Anatoly Burakov:

Quoted 17 lines

Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions [1] because the data itself must live in shared memory.

Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races, and implement the synchronization mechanism based on this lock in the timer library.

Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.

Fixes c0749f7096c7 ("timer: allow management in shared memory")
Quoted 7 lines

[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html

Applied, thanks