timer: fix resource leak in finalize
PATCH 1/1 timer: fix resource leak in finalize
The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
--- lib/librte_timer/rte_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
lib/librte_timer/rte_timer.c +10 -2
@@ -60,6 +60,7 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;
@@ -164,6 +165,7 @@ rte_timer_subsystem_init_v1905(void)if (mz ==NULL )return -EEXIST ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; rte_timer_data_arr[default_data_id].internal_flags |=
@@ -180,6 +182,7 @@ rte_timer_subsystem_init_v1905(void)if (mz ==NULL )return -ENOMEM ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr;for (i =0 ; i <RTE_MAX_DATA_ELS ; i++) {
@@ -205,8 +208,13 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); void __rte_experimentalrte_timer_subsystem_finalize (void) { -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) +return ; + +if (!rte_timer_subsystem_initialized) +return ; + +rte_memzone_free (rte_timer_data_mz); rte_timer_subsystem_initialized =0 ; }
Signature · 1 line
2.6.4
Quoted 42 lines · Erik Gabriel Carrillo
The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
Fixes c0749f7096c7 ("timer: allow management in shared memory")Signed-off-by Erik Gabriel Carrillo <[email protected]>--- lib/librte_timer/rte_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)lib/librte_timer/rte_timer.c +3
-0@@ -60,6 +60,7 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;@@ -164,6 +165,7 @@ rte_timer_subsystem_init_v1905(void)if (mz ==NULL )return -EEXIST ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; rte_timer_data_arr[default_data_id].internal_flags |=@@ -180,6 +182,7 @@ rte_timer_subsystem_init_v1905(void)if (mz ==NULL )return -ENOMEM ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr;for (i =0 ; i <RTE_MAX_DATA_ELS ; i++) {@@ -205,8 +208,13 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); void __rte_experimentalrte_timer_subsystem_finalize (void)
lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217
Quoted 10 lines
lib/librte_timer/rte_timer.c +7
-2{ -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) +return ; + +if (!rte_timer_subsystem_initialized) +return ; + +rte_memzone_free (rte_timer_data_mz);
The patch is a correct fix, but the whole idea of this looks dangerous to me.
If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?
Quoted 4 lines
lib/librte_timer/rte_timer.c +0
-0rte_timer_subsystem_initialized =0 ; }
Signature · 2 lines
Thanks, Anatoly
Quoted 67 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 4:18 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalizeQuoted 27 lines · Erik Gabriel Carrillo
The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
Fixes c0749f7096c7 ("timer: allow management in shared memory")Signed-off-by Erik Gabriel Carrillo <[email protected]>--- lib/librte_timer/rte_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex eb46009..fb7a87e 100644lib/librte_timer/rte_timer.c +1
-0@@ -60,6 +60,7 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id; static uint32_t rte_timer_subsystem_initialized; @@ -164,6 +165,7 @@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return-EEXIST;lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)
Quoted 4 lines
lib/librte_timer/rte_timer.c +1
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; rte_timer_data_arr[default_data_id].internal_flags |= @@ -180,6
Quoted 12 lines
+182,7 @@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return-ENOMEM;lib/librte_timer/rte_timer.c +1
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; for (i = 0; i < RTE_MAX_DATA_ELS; i++) { @@ -205,8 +208,13 @@
BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
void__rte_experimentalrte_timer_subsystem_finalize(void)
{lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217
Quoted 9 lines
lib/librte_timer/rte_timer.c +7
-2-if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) +return ; + +if (!rte_timer_subsystem_initialized) +return ; + +rte_memzone_free (rte_timer_data_mz);The patch is a correct fix, but the whole idea of this looks dangerous to me.
Quoted 3 lines
If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?
Ah yes - that’s right. Perhaps it would be better to maintain a reference count of some sort such that the last process to exit could cause the memzone_free.
Thanks,
Erik
Quoted 9 lines
Quoted 4 lines
rte_timer_subsystem_initialized = 0; }--
Thanks,
Anatoly
Quoted 67 lines · Erik G
Quoted 67 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 4:18 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalizeQuoted 27 lines · Erik Gabriel Carrillo
The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
Fixes c0749f7096c7 ("timer: allow management in shared memory")Signed-off-by Erik Gabriel Carrillo <[email protected]>--- lib/librte_timer/rte_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex eb46009..fb7a87e 100644lib/librte_timer/rte_timer.c +1
-0@@ -60,6 +60,7 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id; static uint32_t rte_timer_subsystem_initialized; @@ -164,6 +165,7 @@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return-EEXIST;lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)
Quoted 4 lines
lib/librte_timer/rte_timer.c +1
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; rte_timer_data_arr[default_data_id].internal_flags |= @@ -180,6
Quoted 12 lines
+182,7 @@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return-ENOMEM;lib/librte_timer/rte_timer.c +1
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; for (i = 0; i < RTE_MAX_DATA_ELS; i++) { @@ -205,8 +208,13 @@BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); void __rte_experimental rte_timer_subsystem_finalize(void) {lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217
Quoted 9 lines
lib/librte_timer/rte_timer.c +7
-2-if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) +return ; + +if (!rte_timer_subsystem_initialized) +return ; + +rte_memzone_free (rte_timer_data_mz);The patch is a correct fix, but the whole idea of this looks dangerous to me.
Quoted 6 lines
Quoted 3 lines
If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?
Ah yes - that’s right. Perhaps it would be better to maintain a reference count of some sort such that the last process to exit could cause the
memzone_free.
It feels like a hack, to be honest. A process can crash or exit without calling rte_eal_cleanup(), which will lead to a memory leak due to refcount being stuck at a value that's not representing reality. It will be saf-er than current approach, but still not ideal.
However, i guess it's a good compromise, if i were to choose between a memory leak and a segfault :D I wonder if there is a better approach.
Quoted 12 lines
Thanks,
ErikQuoted 9 lines
Quoted 4 lines
rte_timer_subsystem_initialized = 0; }--
Thanks,
Anatoly
Signature · 2 lines
Thanks, Anatoly
Quoted 89 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 8:04 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalizeQuoted 51 lines · Erik G
Quoted 51 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Thursday, May 2, 2019 4:18 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [PATCH] timer: fix resource leak in finalizeQuoted 28 lines · Erik Gabriel Carrillo
The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
Fixes c0749f7096c7 ("timer: allow management in shared memory")Signed-off-by Erik Gabriel Carrillo <[email protected]>--- lib/librte_timer/rte_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex eb46009..fb7a87e 100644lib/librte_timer/rte_timer.c +1
-0@@ -60,6 +60,7 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id; static uint32_t rte_timer_subsystem_initialized; @@ -164,6 +165,7@@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return-EEXIST;lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)
Quoted 4 lines
lib/librte_timer/rte_timer.c +1
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; rte_timer_data_arr[default_data_id].internal_flags |= @@ -180,6
Quoted 7 lines
+182,7 @@
rte_timer_subsystem_init_v1905(void)
if (mz == NULL)
return-ENOMEM;lib/librte_timer/rte_timer.c +1
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr;Quoted 16 lines
Quoted 8 lines
Quoted 8 lines
for (i = 0; i < RTE_MAX_DATA_ELS; i++) { @@ -205,8 +208,13 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); void __rte_experimental rte_timer_subsystem_finalize(void) {lib/librte_timer/rte_timer.c +1
-2-if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (rte_eal_process_type () !=RTE_PROC_PRIMARY )Quoted 8 lines
lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 217
Quoted 6 lines
lib/librte_timer/rte_timer.c +6
-0+return ; + +if (!rte_timer_subsystem_initialized) +return ; + +rte_memzone_free (rte_timer_data_mz);The patch is a correct fix, but the whole idea of this looks dangerous to
me.
Quoted 6 lines
Quoted 4 lines
If we exit the primary while secondaries are still running, wouldn't it basically pull out timer data from under secondaries' feet?
Ah yes - that’s right. Perhaps it would be better to maintain a reference
count of some sort such that the last process to exit could cause the
memzone_free.Quoted 1 line
Quoted 7 lines
It feels like a hack, to be honest. A process can crash or exit without calling
rte_eal_cleanup(), which will lead to a memory leak due to refcount being stuck at a value that's not representing reality. It will be saf-er than current approach, but still not ideal.However, i guess it's a good compromise, if i were to choose between a memory leak and a segfault :D I wonder if there is a better approach.
Ok, I will take a look at that approach then as a first step, since a process can already crash before calling rte_eal_cleanup(), which would result in leaks anyway.
Thanks,
Erik
Quoted 18 lines
Quoted 13 lines
Thanks,
ErikQuoted 9 lines
Quoted 4 lines
rte_timer_subsystem_initialized = 0; }--
Thanks,
Anatoly--
Thanks,
Anatoly
PATCH v2 1/1 timer: fix resource leak in finalize
In reply to Erik Gabriel CarrilloThe finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
---
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)
lib/librte_timer/rte_timer.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-)
lib/librte_timer/rte_timer.c +17 -3
@@ -60,6 +60,8 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static rte_atomic16_t *rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;
@@ -155,6 +157,7 @@ rte_timer_subsystem_init_v1905(void)struct rte_timer_data *data;int i, lcore_id;static const char *mz_name ="rte_timer_mz" ; +size_t data_arr_size =RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr);if (rte_timer_subsystem_initialized)return -EALREADY ;
@@ -164,10 +167,14 @@ rte_timer_subsystem_init_v1905(void)if (mz ==NULL )return -EEXIST ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = + (void *)((char *)mz->addr + data_arr_size); rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; +rte_atomic16_inc (rte_timer_mz_refcnt); rte_timer_subsystem_initialized =1 ;
@@ -175,12 +182,15 @@ rte_timer_subsystem_init_v1905(void) } mz =rte_memzone_reserve_aligned (mz_name, -RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr), + data_arr_size +sizeof (*rte_timer_mz_refcnt),SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE );if (mz ==NULL )return -ENOMEM ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size); +rte_atomic16_init (rte_timer_mz_refcnt);for (i =0 ; i <RTE_MAX_DATA_ELS ; i++) { data = &rte_timer_data_arr[i];
@@ -193,6 +203,7 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; +rte_atomic16_inc (rte_timer_mz_refcnt); rte_timer_subsystem_initialized =1 ;
@@ -205,8 +216,11 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); void __rte_experimentalrte_timer_subsystem_finalize (void) { -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (!rte_timer_subsystem_initialized) +return ; + +if (rte_atomic16_dec_and_test (rte_timer_mz_refcnt)) +rte_memzone_free (rte_timer_data_mz); rte_timer_subsystem_initialized =0 ; }
Signature · 1 line
2.6.4
Quoted 29 lines · Erik Gabriel Carrillo
The finalize function should free the memzone created in the init function, rather than freeing the allocation the memzone references, otherwise a memzone descriptor can be leaked.
Fixes c0749f7096c7 ("timer: allow management in shared memory")Signed-off-by Erik Gabriel Carrillo <[email protected]>---
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)lib/librte_timer/rte_timer.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-)lib/librte_timer/rte_timer.c +2
-0@@ -60,6 +60,8 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static rte_atomic16_t *rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;
lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void) · line 160
Quoted 4 lines
lib/librte_timer/rte_timer.c +1
-0@@ -155,6 +157,7 @@ rte_timer_subsystem_init_v1905(void)struct rte_timer_data *data;int i, lcore_id;static const char *mz_name ="rte_timer_mz" ; +size_t data_arr_size =RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr);
nitpicking, but... const?
Quoted 44 lines
lib/librte_timer/rte_timer.c +9
-1if (rte_timer_subsystem_initialized)return -EALREADY ;@@ -164,10 +167,14 @@ rte_timer_subsystem_init_v1905(void)if (mz ==NULL )return -EEXIST ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = + (void *)((char *)mz->addr + data_arr_size); rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; +rte_atomic16_inc (rte_timer_mz_refcnt); rte_timer_subsystem_initialized =1 ;@@ -175,12 +182,15 @@ rte_timer_subsystem_init_v1905(void) } mz =rte_memzone_reserve_aligned (mz_name, -RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr), + data_arr_size +sizeof (*rte_timer_mz_refcnt),SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE );if (mz ==NULL )return -ENOMEM ; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size); +rte_atomic16_init (rte_timer_mz_refcnt);for (i =0 ; i <RTE_MAX_DATA_ELS ; i++) { data = &rte_timer_data_arr[i];@@ -193,6 +203,7 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; +rte_atomic16_inc (rte_timer_mz_refcnt); rte_timer_subsystem_initialized =1 ;
lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223
Quoted 10 lines
lib/librte_timer/rte_timer.c +5
-2@@ -205,8 +216,11 @@ BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); void __rte_experimentalrte_timer_subsystem_finalize (void) { -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (!rte_timer_subsystem_initialized) +return ; + +if (rte_atomic16_dec_and_test (rte_timer_mz_refcnt)) +rte_memzone_free (rte_timer_data_mz);
I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also support a case when secondary initializes after primary stops running.
Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.
I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.
Quoted 4 lines
lib/librte_timer/rte_timer.c +0
-0rte_timer_subsystem_initialized =0 ; }
Signature · 2 lines
Thanks, Anatoly
Hi Anatoly,
Thanks for the review. Comments in-line:
<...snipped...>
Quoted 6 lines
Quoted 6 lines
#define RTE_MAX_DATA_ELS 64(quoted diff) +2
-0+static const struct rte_memzone *rte_timer_data_mz; static +rte_atomic16_t *rte_timer_mz_refcnt; static struct rte_timer_data *rte_timer_data_arr; static const uint32_t default_data_id; static uint32_t rte_timer_subsystem_initialized; @@ -155,6 +157,7 @@
Quoted 9 lines
Review
Quoted 6 lines
rte_timer_subsystem_init_v1905(void)
structrte_timer_data*data;
int i,lcore_id;
static const char*mz_name= "rte_timer_mz";(quoted diff) +2
-0+ size_t data_arr_size = RTE_MAX_DATA_ELS * +sizeof(*rte_timer_data_arr);nitpicking, but... const?
No problem - I'll make this change if this line persists into the next version.
<...snipped...>
Quoted 16 lines
Review
Quoted 2 lines
(quoted diff) +0
-0@@ -205,8 +216,11 @@
BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223
Quoted 10 lines
lib/librte_timer/rte_timer.c +5
-2void __rte_experimentalrte_timer_subsystem_finalize (void) { -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (!rte_timer_subsystem_initialized) +return ; + +if (rte_atomic16_dec_and_test (rte_timer_mz_refcnt)) +rte_memzone_free (rte_timer_data_mz);I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also
Indeed, thanks for catching this.
Quoted 10 lines
support a case when secondary initializes after primary stops running.
Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking
rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.
I don't see a solution here currently either. I'll look at removing the memzone_free() call and possibly the rte_timer_subsystem_finalize() API, since it seems like there's no reason for it to exist if it can't free the allocations.
Regards,
Erik
Quoted 9 lines
lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
Quoted 4 lines
lib/librte_timer/rte_timer.c +0
-0rte_timer_subsystem_initialized =0 ; }--
Thanks,
Anatoly
Quoted 52 lines · Erik G
Hi Anatoly,
Thanks for the review. Comments in-line:
<...snipped...>
Quoted 6 lines
Quoted 6 lines
#define RTE_MAX_DATA_ELS 64(quoted diff) +2
-0+static const struct rte_memzone *rte_timer_data_mz; static +rte_atomic16_t *rte_timer_mz_refcnt; static struct rte_timer_data *rte_timer_data_arr; static const uint32_t default_data_id; static uint32_t rte_timer_subsystem_initialized; @@ -155,6 +157,7 @@Quoted 9 lines
Review
Quoted 6 lines
rte_timer_subsystem_init_v1905(void)struct rte_timer_data *data; int i, lcore_id; static const char *mz_name = "rte_timer_mz";(quoted diff) +2
-0+ size_t data_arr_size = RTE_MAX_DATA_ELS * +sizeof(*rte_timer_data_arr);nitpicking, but... const?
No problem - I'll make this change if this line persists into the next version.
<...snipped...>
Quoted 16 lines
Review
Quoted 2 lines
(quoted diff) +0
-0@@ -205,8 +216,11 @@
BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223
Quoted 10 lines
lib/librte_timer/rte_timer.c +5
-2void __rte_experimentalrte_timer_subsystem_finalize (void) { -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (!rte_timer_subsystem_initialized) +return ; + +if (rte_atomic16_dec_and_test (rte_timer_mz_refcnt)) +rte_memzone_free (rte_timer_data_mz);I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also
Indeed, thanks for catching this.
Quoted 7 lines
support a case when secondary initializes after primary stops running.
Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking
rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.
Quoted 7 lines
Quoted 3 lines
I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.
I don't see a solution here currently either. I'll look at removing the
memzone_free()call and possibly therte_timer_subsystem_finalize()API, since it seems like there's no reason for it to exist if it can't free the allocations.
I wonder if there are other places in DPDK where this pattern is used.
Technically, this kind of thing /could/ be resolved by having something in our multiprocess shared memory outside of DPDK heap. I.e. store something in rte_eal_memconfig like some other things do. This change, however, would require an ABI break, so while changing this particular API won't need a deprecation notice, the change itself would.
Quoted 13 lines
Regards,
ErikQuoted 9 lines
lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
Quoted 4 lines
lib/librte_timer/rte_timer.c +0
-0rte_timer_subsystem_initialized =0 ; }--
Thanks,
Anatoly
Signature · 2 lines
Thanks, Anatoly
Quoted 74 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Wednesday, May 8, 2019 3:50 AM
To: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [PATCH v2] timer: fix resource leak in finalizeQuoted 54 lines · Erik G
Hi Anatoly,
Thanks for the review. Comments in-line:
<...snipped...>
Quoted 7 lines
Quoted 7 lines
#define RTE_MAX_DATA_ELS 64(quoted diff) +2
-0+static const struct rte_memzone *rte_timer_data_mz; static +rte_atomic16_t *rte_timer_mz_refcnt; static struct rte_timer_data *rte_timer_data_arr; static const uint32_t default_data_id; static uint32_t rte_timer_subsystem_initialized; @@ -155,6 +157,7@@
Quoted 9 lines
Review
Quoted 6 lines
rte_timer_subsystem_init_v1905(void)struct rte_timer_data *data; int i, lcore_id; static const char *mz_name = "rte_timer_mz";(quoted diff) +2
-0+ size_t data_arr_size = RTE_MAX_DATA_ELS * +sizeof(*rte_timer_data_arr);nitpicking, but... const?
No problem - I'll make this change if this line persists into the next version.
<...snipped...>
Quoted 17 lines
Review
Quoted 2 lines
(quoted diff) +0
-0@@ -205,8 +216,11 @@
BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05); · line 223
Quoted 10 lines
lib/librte_timer/rte_timer.c +5
-2void __rte_experimentalrte_timer_subsystem_finalize (void) { -if (rte_timer_data_arr) -rte_free (rte_timer_data_arr); +if (!rte_timer_subsystem_initialized) +return ; + +if (rte_atomic16_dec_and_test (rte_timer_mz_refcnt)) +rte_memzone_free (rte_timer_data_mz);I think there's a race here. You may get preempted after test but before free, where another secondary could initialize. As far as i know, we also
Indeed, thanks for catching this.
Quoted 7 lines
support a case when secondary initializes after primary stops running.
Let's even suppose that we allow secondary processes to initialize the timer subsystem by reserving memzone and checking
rte_errno. You would still have a chance of two init/deinit conflicting, because there's a hole between memzone allocation and atomic increment.Quoted 7 lines
Quoted 3 lines
I don't think this race can be resolved in a safe way, so we might just have to settle for a memory leak.
I don't see a solution here currently either. I'll look at removing the
memzone_free()call and possibly therte_timer_subsystem_finalize()API, since it seems like there's no reasonfor it to exist if it can't free the allocations.
Quoted 7 lines
I wonder if there are other places in DPDK where this pattern is used.
Technically, this kind of thing /could/ be resolved by having something in our multiprocess shared memory outside of DPDK heap. I.e. store something in
rte_eal_memconfiglike some other things do. This change, however, would require an ABI break, so while changing this particular API won't need a deprecation notice, the change itself would.
I like this idea; thanks for the suggestion. I went ahead and submitted a new version of this patch with this approach. It's dependent on one other new patch that allows secondary processes to reserve the memzone. I also submitted a deprecation notice for the ABI break for the change to rte_mem_config.
Thomas, I suspect that I should mark v3 of this patch as "deferred", so I've gone ahead and done that, but let me know if that's wrong.
Thanks,
Erik
Quoted 19 lines
Quoted 14 lines
Regards,
ErikQuoted 9 lines
lib/librte_timer/rte_timer.c BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
Quoted 4 lines
lib/librte_timer/rte_timer.c +0
-0rte_timer_subsystem_initialized =0 ; }--
Thanks,
Anatoly--
Thanks,
Anatoly
09/05/2019 01:01, Carrillo, Erik G:
Quoted 7 lines
I like this idea; thanks for the suggestion. I went ahead and submitted a new version of this patch with this approach. It's dependent on one other new patch that allows secondary processes to reserve the memzone. I also submitted a deprecation notice for the ABI break for the change to
rte_mem_config.Thomas, I suspect that I should mark v3 of this patch as "deferred", so I've gone ahead and done that, but let me know if that's wrong.
Any patch targetting 19.08 should be marked as deferred, thanks.
PATCH v3 1/1 timer: fix resource leak in finalize
In reply to Erik Gabriel CarrilloBy using a lock added to the rte_mem_config (which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.
---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)
This patch depends on http://patches.dpdk.org/patch/53333/.
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)
lib/librte_eal/common/include/rte_eal_memconfig.h | 3 +++ lib/librte_timer/rte_timer.c | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-)
lib/librte_eal/common/include/rte_eal_memconfig.h +3 -0
@@ -64,6 +64,9 @@ struct rte_mem_config {rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */ +rte_spinlock_t timer_subsystem_lock; +/**< indicates whether timer subsystem init/finalize is in progress. */ +/* memory segments and zones */ struct rte_fbarray memzones;/**< Memzone descriptors. */
lib/librte_timer/rte_timer.c +22 -1
@@ -26,6 +26,7 @@#include <rte_malloc.h> #include <rte_compat.h> #include <rte_errno.h> +#include <rte_eal_memconfig.h> #include "rte_timer.h"
@@ -60,7 +61,12 @@ struct rte_timer_data {uint8_t internal_flags; }; +#define RTE_TIMER_SUBSYSTEM_LOCK \ + (&rte_eal_get_configuration()->mem_config->timer_subsystem_lock) +#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static int *rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;
@@ -163,22 +169,27 @@ rte_timer_subsystem_init_v1905(void)if (rte_timer_subsystem_initialized)return -EALREADY ; +rte_spinlock_lock (RTE_TIMER_SUBSYSTEM_LOCK ); lookup: mz =rte_memzone_lookup (mz_name);if (mz ==NULL ) { - mz =rte_memzone_reserve_aligned (mz_name, data_arr_size, + mz =rte_memzone_reserve_aligned (mz_name, + data_arr_size +sizeof (*rte_timer_mz_refcnt),SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE );if (mz ==NULL ) {if (rte_errno ==EEXIST ) goto lookup; +rte_spinlock_unlock (RTE_TIMER_SUBSYSTEM_LOCK );return -ENOMEM ; } do_full_init = true; } + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);if (do_full_init) {for (i =0 ; i <RTE_MAX_DATA_ELS ; i++) {
@@ -195,6 +206,9 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; + (*rte_timer_mz_refcnt)++; + +rte_spinlock_unlock (RTE_TIMER_SUBSYSTEM_LOCK ); rte_timer_subsystem_initialized =1 ;
@@ -210,6 +224,13 @@ rte_timer_subsystem_finalize(void)if (!rte_timer_subsystem_initialized)return ; +rte_spinlock_lock (RTE_TIMER_SUBSYSTEM_LOCK ); + +if (--(*rte_timer_mz_refcnt) ==0 ) +rte_memzone_free (rte_timer_data_mz); + +rte_spinlock_unlock (RTE_TIMER_SUBSYSTEM_LOCK ); + rte_timer_subsystem_initialized =0 ; }
Signature · 1 line
2.6.4
Quoted 27 lines · Erik Gabriel Carrillo
By using a lock added to the
rte_mem_config(which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.Signed-off-by Erik Gabriel Carrillo <[email protected]>---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)This patch depends on http://patches.dpdk.org/patch/53333/.
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)lib/librte_eal/common/include/rte_eal_memconfig.h | 3 +++ lib/librte_timer/rte_timer.c | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-)lib/librte_eal/common/include/rte_eal_memconfig.h +0
-0
lib/librte_eal/common/include/rte_eal_memconfig.h struct rte_mem_config { · line 69
Quoted 6 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +3
-0@@ -64,6 +64,9 @@ struct rte_mem_config {rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */ +rte_spinlock_t timer_subsystem_lock; +/**< indicates whether timer subsystem init/finalize is in progress. */ +
I believe there's an initialize function somewhere which will initialize all of these locks. This lock should be in there as well.
Other than that, i'm OK with this change, however i feel like /just/ adding this would be a missed opportunity, because next time we want to add something here it will be an ABI break again.
We could perhaps use this opportunity to leave some padding for future data. I'm not sure how would that look like, just an idea floating in my head :)
Signature · 2 lines
Thanks, Anatoly
09/05/2019 10:29, Burakov, Anatoly:
Quoted 38 lines · Erik Gabriel Carrillo
Quoted 27 lines
By using a lock added to the
rte_mem_config(which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.Signed-off-by Erik Gabriel Carrillo <[email protected]>---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)This patch depends on http://patches.dpdk.org/patch/53333/.
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)lib/librte_eal/common/include/rte_eal_memconfig.h | 3 +++ lib/librte_timer/rte_timer.c | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-)lib/librte_eal/common/include/rte_eal_memconfig.h +0
-0lib/librte_eal/common/include/rte_eal_memconfig.h struct rte_mem_config { · line 69
Quoted 6 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +3
-0@@ -64,6 +64,9 @@ struct rte_mem_config {rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */ +rte_spinlock_t timer_subsystem_lock; +/**< indicates whether timer subsystem init/finalize is in progress. */ +I believe there's an initialize function somewhere which will initialize all of these locks. This lock should be in there as well.
Quoted 7 lines
Other than that, i'm OK with this change, however i feel like /just/ adding this would be a missed opportunity, because next time we want to add something here it will be an ABI break again.
We could perhaps use this opportunity to leave some padding for future data. I'm not sure how would that look like, just an idea floating in my head :)
Any update or other opinion?
Quoted 43 lines · Thomas Monjalon
09/05/2019 10:29, Burakov, Anatoly:
Quoted 42 lines · Erik Gabriel Carrillo
Quoted 27 lines
By using a lock added to the
rte_mem_config(which lives in shared memory), we can synchronize multiple processes in init/finalize and safely free allocations made during init.Signed-off-by Erik Gabriel Carrillo <[email protected]>---
changes in v3:
- The previous version had race condition. This version fixes the race by adding a lock in shared memory outside of the DPDK heap area that can be used to safely free the memzone reserved in the subsystem init call. (Anatoly)This patch depends on http://patches.dpdk.org/patch/53333/.
changes in v2:
- Handle scenario where primary process exits before secondaries such that memzone is not freed early (Anatoly)lib/librte_eal/common/include/rte_eal_memconfig.h | 3 +++ lib/librte_timer/rte_timer.c | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-)lib/librte_eal/common/include/rte_eal_memconfig.h +0
-0lib/librte_eal/common/include/rte_eal_memconfig.h struct rte_mem_config { · line 69
Quoted 6 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +3
-0@@ -64,6 +64,9 @@ struct rte_mem_config {rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */ +rte_spinlock_t timer_subsystem_lock; +/**< indicates whether timer subsystem init/finalize is in progress. */ +I believe there's an initialize function somewhere which will initialize all of these locks. This lock should be in there as well.
Other than that, i'm OK with this change, however i feel like /just/ adding this would be a missed opportunity, because next time we want to add something here it will be an ABI break again.
Quoted 6 lines
Quoted 3 lines
We could perhaps use this opportunity to leave some padding for future data. I'm not sure how would that look like, just an idea floating in my head :)
Any update or other opinion?
I have a patchset that hides mem config - if we are to add a lock, it would be after adding that patch, as a bugfix (since by then it would n longer be an ABI breakage or an API change).
Signature · 2 lines
Thanks, Anatoly
cover Fix timer resource leak
In reply to Erik Gabriel CarrilloPrevious attempts [1] at fixing the resource leak have been deemed unsuccessful because of limitations around what can be done without breaking the ABI. Now that we've broken the EAL ABI, we can fix this issue properly.
This patchset is adding a new lock API, as well as fixes the actual issue.
The patchset dependson the FreeBSD fixes [2], as well as the mem config patchset [3].
[1] http://patches.dpdk.org/patch/53334/
[2] http://patches.dpdk.org/project/dpdk/list/?series=5161
[3] http://patches.dpdk.org/project/dpdk/list/?series=5162
Anatoly Burakov (2):
eal: add internal locks for timer lib into EAL
timer: fix resource leak in finalize
lib/librte_eal/common/eal_common_mcfg.c | 14 +++++++ lib/librte_eal/common/eal_memcfg.h | 2 + .../common/include/rte_eal_memconfig.h | 18 ++++++++ lib/librte_eal/rte_eal_version.map | 2 + lib/librte_timer/rte_timer.c | 41 +++++++++++++------ lib/librte_timer/rte_timer.h | 5 ++- 6 files changed, 67 insertions(+), 15 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/2 eal: add internal locks for timer lib into EAL
Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html
--- lib/librte_eal/common/eal_common_mcfg.c | 14 ++++++++++++++ lib/librte_eal/common/eal_memcfg.h | 2 ++ .../common/include/rte_eal_memconfig.h | 18 ++++++++++++++++++ lib/librte_eal/rte_eal_version.map | 2 ++ 4 files changed, 36 insertions(+)
lib/librte_eal/common/eal_common_mcfg.c +14 -0
@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config;rte_rwlock_write_unlock (&mcfg->mplock); } + +void +rte_mcfg_timer_lock (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +rte_spinlock_lock (&mcfg->tlock); +} + +void +rte_mcfg_timer_unlock (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +rte_spinlock_unlock (&mcfg->tlock); +}
lib/librte_eal/common/eal_memcfg.h +2 -0
@@ -11,6 +11,7 @@#include <rte_memory.h> #include <rte_memzone.h> #include <rte_pause.h> +#include <rte_spinlock.h> #include <rte_rwlock.h> #include <rte_tailq.h>
@@ -38,6 +39,7 @@ struct rte_mem_config {rte_rwlock_t mlock;/**< only used by memzone LIB for thread-safe. */ rte_rwlock_t qlock;/**< used for tailq operation for thread safe. */ rte_rwlock_t mplock;/**< only used by mempool LIB for thread-safe. */ +rte_spinlock_t tlock;/**< needed for timer lib thread safety. */ rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */
lib/librte_eal/common/include/rte_eal_memconfig.h +18 -0
@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);void rte_mcfg_mempool_write_unlock (void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Lock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental +rte_mcfg_timer_lock (void); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Unlock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental +rte_mcfg_timer_unlock (void); + #ifdef __cplusplus }#endif
lib/librte_eal/rte_eal_version.map +2 -0
@@ -367,6 +367,8 @@ EXPERIMENTAL { rte_malloc_heap_memory_detach; rte_malloc_heap_memory_remove; rte_malloc_heap_socket_is_external; + rte_mcfg_timer_lock; + rte_mcfg_timer_unlock; rte_mem_alloc_validator_register; rte_mem_alloc_validator_unregister; rte_mem_check_dma_mask;
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH 1/2] eal: add internal locks for timer lib into EAL
Hi Anatoly,
Quoted 18 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Tuesday, June 25, 2019 11:12 AM
To: [email protected]
Cc: Carrillo, Erik G <[email protected]>
Subject: [PATCH 1/2] eal: add internal locks for timer lib into EALCurrently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.htmlSigned-off-by Anatoly Burakov <[email protected]>
<... snipped ...>
lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 118
Quoted 13 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7
-0@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);void rte_mcfg_mempool_write_unlock (void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Lock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental
Depending on the decision made for the following thread, the "__rte_experimental" tag location may move:
https://mails.dpdk.org/archives/dev/2019-June/136050.html
Quoted 4 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +4
-0+rte_mcfg_timer_lock (void); + +/** +* @warning
lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 132
Quoted 10 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7
-0+* @b EXPERIMENTAL: this API may change without prior notice +* +* Unlock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental +rte_mcfg_timer_unlock (void); + #ifdef __cplusplus }#endif
<... snipped ...>
Other than that:
Re: [dpdk-dev] [PATCH 1/2] eal: add internal locks for timer lib into EAL
In reply to Anatoly BurakovQuoted 77 lines · Anatoly Burakov
Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.htmlSigned-off-by Anatoly Burakov <[email protected]>--- lib/librte_eal/common/eal_common_mcfg.c | 14 ++++++++++++++ lib/librte_eal/common/eal_memcfg.h | 2 ++ .../common/include/rte_eal_memconfig.h | 18 ++++++++++++++++++ lib/librte_eal/rte_eal_version.map | 2 ++ 4 files changed, 36 insertions(+)lib/librte_eal/common/eal_common_mcfg.c +5
-0@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config;rte_rwlock_write_unlock (&mcfg->mplock); } + +void +rte_mcfg_timer_lock (void ) +{ + struct rte_mem_config *mcfg =rte_eal_get_configuration()->mem_config;
lib/librte_eal/common/eal_common_mcfg.c +7
-0+rte_spinlock_lock (&mcfg->tlock); +} + +void +rte_mcfg_timer_unlock (void ) +{ + struct rte_mem_config *mcfg =rte_eal_get_configuration()->mem_config;
lib/librte_eal/common/eal_common_mcfg.c +2
-0+rte_spinlock_unlock (&mcfg->tlock); +}lib/librte_eal/common/eal_memcfg.h +1
-0@@ -11,6 +11,7 @@#include <rte_memory.h> #include <rte_memzone.h> #include <rte_pause.h> +#include <rte_spinlock.h> #include <rte_rwlock.h> #include <rte_tailq.h> @@ -38,6 +39,7 @@ struct rte_mem_config {rte_rwlock_t mlock;/**< only used by memzone LIB for thread-safe. */
rte_rwlock_tqlock; /**< used for tailq operation for thread
safe. */
rte_rwlock_tmplock; /**< only used by mempool LIB for
thread-safe. */lib/librte_eal/common/eal_memcfg.h +1
-0+rte_spinlock_t tlock;/**< needed for timer lib thread safety. */ rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */ lib/librte_eal/common/include/rte_eal_memconfig.h +0
-0@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);void
lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 119
Quoted 10 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7
-0rte_mcfg_mempool_write_unlock (void); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Lock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental
As mentionned by Erik,
rte_experimental
void
+rte_mcfg_timer_lock(void);
lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void);
Quoted 9 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +8
-0+ +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Unlock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental
rte_experimental
void
Quoted 20 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +2
-0+rte_mcfg_timer_unlock (void); + #ifdef __cplusplus }#endif lib/librte_eal/rte_eal_version.map +2
-1@@ -367,6 +367,8 @@ EXPERIMENTAL { rte_malloc_heap_memory_detach; rte_malloc_heap_memory_remove; rte_malloc_heap_socket_is_external; + rte_mcfg_timer_lock; + rte_mcfg_timer_unlock; rte_mem_alloc_validator_register; rte_mem_alloc_validator_unregister; rte_mem_check_dma_mask; --
Quoted 2 lines
2.17.1
Can you put this at the end of the EXPERIMENTAL block under the 19.08 comment?
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH 1/2] eal: add internal locks for timer lib into EAL
Quoted 129 lines · David Marchand
Quoted 77 lines · Anatoly Burakov
Currently, timer library has a memory leak because there is no way to concurrently initialize/deinitialize shared memory because of race conditions [1].
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races.
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.htmlSigned-off-by Anatoly Burakov <[email protected]>--- lib/librte_eal/common/eal_common_mcfg.c | 14 ++++++++++++++ lib/librte_eal/common/eal_memcfg.h | 2 ++ .../common/include/rte_eal_memconfig.h | 18 ++++++++++++++++++ lib/librte_eal/rte_eal_version.map | 2 ++ 4 files changed, 36 insertions(+)lib/librte_eal/common/eal_common_mcfg.c +5
-0@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config;rte_rwlock_write_unlock (&mcfg->mplock); } + +void +rte_mcfg_timer_lock (void ) +{ + struct rte_mem_config *mcfg =rte_eal_get_configuration()->mem_config;
lib/librte_eal/common/eal_common_mcfg.c +7
-0+rte_spinlock_lock (&mcfg->tlock); +} + +void +rte_mcfg_timer_unlock (void ) +{ + struct rte_mem_config *mcfg =rte_eal_get_configuration()->mem_config;
lib/librte_eal/common/eal_common_mcfg.c +2
-0+rte_spinlock_unlock (&mcfg->tlock); +}lib/librte_eal/common/eal_memcfg.h +1
-0@@ -11,6 +11,7 @@#include <rte_memory.h> #include <rte_memzone.h> #include <rte_pause.h> +#include <rte_spinlock.h> #include <rte_rwlock.h> #include <rte_tailq.h> @@ -38,6 +39,7 @@ struct rte_mem_config {rte_rwlock_t mlock;/**< only used by memzone LIB for thread-safe. */
rte_rwlock_tqlock; /**< used for tailq operation for thread
safe. */
rte_rwlock_tmplock; /**< only used by mempool LIB for
thread-safe. */lib/librte_eal/common/eal_memcfg.h +1
-0+rte_spinlock_t tlock;/**< needed for timer lib thread safety. */ rte_rwlock_t memory_hotplug_lock;/**< indicates whether memory hotplug request is in progress. */ lib/librte_eal/common/include/rte_eal_memconfig.h +0
-0@@ -109,6 +109,24 @@ rte_mcfg_mempool_write_lock(void);void lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void); · line 119
Quoted 10 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +7
-0rte_mcfg_mempool_write_unlock (void); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Lock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimentalAs mentionned by Erik,
rte_experimental
void+rte_mcfg_timer_lock(void);
lib/librte_eal/common/include/rte_eal_memconfig.h rte_mcfg_mempool_write_lock(void);
Quoted 9 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +8
-0+ +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Unlock the internal EAL Timer Library lock for exclusive access. +*/ +void __rte_experimental
rte_experimental
voidQuoted 17 lines
lib/librte_eal/common/include/rte_eal_memconfig.h +2
-0+rte_mcfg_timer_unlock (void); + #ifdef __cplusplus }#endif lib/librte_eal/rte_eal_version.map +2
-0@@ -367,6 +367,8 @@ EXPERIMENTAL { rte_malloc_heap_memory_detach; rte_malloc_heap_memory_remove; rte_malloc_heap_socket_is_external; + rte_mcfg_timer_lock; + rte_mcfg_timer_unlock; rte_mem_alloc_validator_register;
Quoted 9 lines
Quoted 5 lines
lib/librte_eal/rte_eal_version.map +0
-1rte_mem_alloc_validator_unregister; rte_mem_check_dma_mask; --2.17.1
Can you put this at the end of the EXPERIMENTAL block under the 19.08 comment?
OK, will do both for v2.
Signature · 2 lines
Thanks, Anatoly
PATCH 2/2 timer: fix resource leak in finalize
In reply to Anatoly BurakovCurrently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.
However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
--- lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------ lib/librte_timer/rte_timer.h | 5 +++-- 2 files changed, 31 insertions(+), 15 deletions(-)
lib/librte_timer/rte_timer.c +28 -13
@@ -13,6 +13,7 @@#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> +#include <rte_eal_memconfig.h> #include <rte_per_lcore.h> #include <rte_memory.h> #include <rte_launch.h>
@@ -61,6 +62,8 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static int *volatile rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;
@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)int i, lcore_id;static const char *mz_name ="rte_timer_mz" ;const size_t data_arr_size = -RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +const size_t mem_size = data_arr_size +sizeof (*rte_timer_mz_refcnt);bool do_full_init = true;if (rte_timer_subsystem_initialized)return -EALREADY ; -reserve: - rte_errno =0 ; - mz =rte_memzone_reserve_aligned (mz_name, data_arr_size,SOCKET_ID_ANY , -0 ,RTE_CACHE_LINE_SIZE ); +rte_mcfg_timer_lock (); + + mz =rte_memzone_lookup (mz_name);if (mz ==NULL ) { -if (rte_errno ==EEXIST ) { - mz =rte_memzone_lookup (mz_name); -if (mz ==NULL ) - goto reserve; - - do_full_init = false; - }else + mz =rte_memzone_reserve_aligned (mz_name, mem_size, +SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE ); +if (mz ==NULL ) { +rte_mcfg_timer_unlock ();return -ENOMEM ; - } + } + do_full_init = true; + }else + do_full_init = false; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);if (do_full_init) {for (i =0 ; i <RTE_MAX_DATA_ELS ; i++) {
@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; + (*rte_timer_mz_refcnt)++; + +rte_mcfg_timer_unlock (); rte_timer_subsystem_initialized =1 ;
@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)if (!rte_timer_subsystem_initialized)return ; +rte_mcfg_timer_lock (); + +if (--(*rte_timer_mz_refcnt) ==0 ) +rte_memzone_free (rte_timer_data_mz); + +rte_mcfg_timer_unlock (); + rte_timer_subsystem_initialized =0 ; }
lib/librte_timer/rte_timer.h +3 -2
@@ -170,10 +170,11 @@ int __rte_experimental rte_timer_data_dealloc(uint32_t id);* Initializes internal variables (list, locks and so on) for the RTE * timer library. * +* @note +* This function must be called in every process before using the library. +* * @return * - 0: Success -* - -EEXIST: Returned in secondary process when primary process has not -* yet initialized the timer subsystem * - -ENOMEM: Unable to allocate memory needed to initialize timer * subsystem */
Signature · 1 line
2.17.1
Quoted 19 lines
-----Original Message-----
From: Burakov, Anatoly
Sent: Tuesday, June 25, 2019 11:12 AM
To: [email protected]
Cc: Robert Sanford <[email protected]>; Carrillo, Erik G <[email protected]>
Subject: [PATCH 2/2] timer: fix resource leak in finalizeCurrently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.
However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.
Quoted 6 lines
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
Signed-off-by Erik Gabriel Carrillo <[email protected]>Signed-off-by Anatoly Burakov <[email protected]>
Quoted 130 lines · Anatoly Burakov
Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.
However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
Signed-off-by Erik Gabriel Carrillo <[email protected]>Signed-off-by Anatoly Burakov <[email protected]>--- lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------ lib/librte_timer/rte_timer.h | 5 +++-- 2 files changed, 31 insertions(+), 15 deletions(-)lib/librte_timer/rte_timer.c +18
-13@@ -13,6 +13,7 @@#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> +#include <rte_eal_memconfig.h> #include <rte_per_lcore.h> #include <rte_memory.h> #include <rte_launch.h> @@ -61,6 +62,8 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static int *volatile rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)int i, lcore_id;static const char *mz_name ="rte_timer_mz" ;const size_t data_arr_size = -RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +const size_t mem_size = data_arr_size +sizeof (*rte_timer_mz_refcnt);bool do_full_init = true;if (rte_timer_subsystem_initialized)return -EALREADY ; -reserve: - rte_errno =0 ; - mz =rte_memzone_reserve_aligned (mz_name, data_arr_size,SOCKET_ID_ANY , -0 ,RTE_CACHE_LINE_SIZE ); +rte_mcfg_timer_lock (); + + mz =rte_memzone_lookup (mz_name);if (mz ==NULL ) { -if (rte_errno ==EEXIST ) { - mz =rte_memzone_lookup (mz_name); -if (mz ==NULL ) - goto reserve; - - do_full_init = false; - }else + mz =rte_memzone_reserve_aligned (mz_name, mem_size, +SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE ); +if (mz ==NULL ) { +rte_mcfg_timer_unlock ();return -ENOMEM ; - } + } + do_full_init = true; + }else + do_full_init = false; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);if (do_full_init) { for (i = 0; i < RTE_MAX_DATA_ELS; i++) {lib/librte_timer/rte_timer.c +10
-0@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; + (*rte_timer_mz_refcnt)++; + +rte_mcfg_timer_unlock (); rte_timer_subsystem_initialized =1 ;@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)if (!rte_timer_subsystem_initialized)return ; +rte_mcfg_timer_lock (); + +if (--(*rte_timer_mz_refcnt) ==0 ) +rte_memzone_free (rte_timer_data_mz); + +rte_mcfg_timer_unlock (); + rte_timer_subsystem_initialized =0 ; }lib/librte_timer/rte_timer.h +2
-0@@ -170,10 +170,11 @@ int __rte_experimental rte_timer_data_dealloc(uint32_t id); * Initializes internal variables (list, locks and so on) for the RTE * timer library. * + * @note + * This function must be called in every process before using thelibrary.
+ *
* @return
Quoted 9 lines
* - 0: Success
- * --EEXIST: Returned in secondary process when primary process has not
- * yet initialized the timer subsystem
* --ENOMEM: Unable to allocate memory needed to initialize timer
* subsystem
*/
--
2.17.1
Can be squashed with the first patch.
Signature · 1 line
David Marchand
Quoted 136 lines · David Marchand
Quoted 133 lines · Anatoly Burakov
Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.
However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
Signed-off-by Erik Gabriel Carrillo <[email protected]>Signed-off-by Anatoly Burakov <[email protected]>--- lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------ lib/librte_timer/rte_timer.h | 5 +++-- 2 files changed, 31 insertions(+), 15 deletions(-)lib/librte_timer/rte_timer.c +18
-13@@ -13,6 +13,7 @@#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> +#include <rte_eal_memconfig.h> #include <rte_per_lcore.h> #include <rte_memory.h> #include <rte_launch.h> @@ -61,6 +62,8 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static int *volatile rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)int i, lcore_id;static const char *mz_name ="rte_timer_mz" ;const size_t data_arr_size = -RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +const size_t mem_size = data_arr_size +sizeof (*rte_timer_mz_refcnt);bool do_full_init = true;if (rte_timer_subsystem_initialized)return -EALREADY ; -reserve: - rte_errno =0 ; - mz =rte_memzone_reserve_aligned (mz_name, data_arr_size,SOCKET_ID_ANY , -0 ,RTE_CACHE_LINE_SIZE ); +rte_mcfg_timer_lock (); + + mz =rte_memzone_lookup (mz_name);if (mz ==NULL ) { -if (rte_errno ==EEXIST ) { - mz =rte_memzone_lookup (mz_name); -if (mz ==NULL ) - goto reserve; - - do_full_init = false; - }else + mz =rte_memzone_reserve_aligned (mz_name, mem_size, +SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE ); +if (mz ==NULL ) { +rte_mcfg_timer_unlock ();return -ENOMEM ; - } + } + do_full_init = true; + }else + do_full_init = false; + rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);if (do_full_init) { for (i = 0; i < RTE_MAX_DATA_ELS; i++) {lib/librte_timer/rte_timer.c +10
-0@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; + (*rte_timer_mz_refcnt)++; + +rte_mcfg_timer_unlock (); rte_timer_subsystem_initialized =1 ;@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)if (!rte_timer_subsystem_initialized)return ; +rte_mcfg_timer_lock (); + +if (--(*rte_timer_mz_refcnt) ==0 ) +rte_memzone_free (rte_timer_data_mz); + +rte_mcfg_timer_unlock (); + rte_timer_subsystem_initialized =0 ; }lib/librte_timer/rte_timer.h +2
-0@@ -170,10 +170,11 @@ int __rte_experimental rte_timer_data_dealloc(uint32_t id); * Initializes internal variables (list, locks and so on) for the RTE * timer library. * + * @note + * This function must be called in every process before using thelibrary.
+ *
* @return
* - 0: Success
- * --EEXIST: Returned in secondary process when primary process has not
- * yet initialized the timer subsystem
Quoted 9 lines
Quoted 6 lines
* -
-ENOMEM: Unable to allocate memory needed to initialize timer
* subsystem
*/
--
2.17.1Can be squashed with the first patch.
This is not just search-and-replace - this is also a bugfix. I can squash the search-and-replace part with the first patch, but this commit will have to stay IMO.
Signature · 2 lines
Thanks, Anatoly
Quoted 148 lines · Anatoly
Quoted 1 line · David Marchand
On Tue, Jun 25, 2019 at 6:12 PM Anatoly Burakov <
Quoted 88 lines
Quoted 86 lines · wrote:
Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions because the data itself must live in shared memory.
However, there is now a timer library lock in the shared memory config, which can be used to synchronize access to the timer library shared memory. Use it to initialize/deinitialize timer library shared data in a safe way. There is still a way to leak the memory (by killing one of the processes), but we can't do anything about that.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
Signed-off-by Erik Gabriel Carrillo <[email protected]>Signed-off-by Anatoly Burakov <[email protected]>--- lib/librte_timer/rte_timer.c | 41 ++++++++++++++++++++++++------------ lib/librte_timer/rte_timer.h | 5 +++-- 2 files changed, 31 insertions(+), 15 deletions(-)lib/librte_timer/rte_timer.c +16
-13@@ -13,6 +13,7 @@#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> +#include <rte_eal_memconfig.h> #include <rte_per_lcore.h> #include <rte_memory.h> #include <rte_launch.h> @@ -61,6 +62,8 @@ struct rte_timer_data { };#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static int *volatile rte_timer_mz_refcnt;static struct rte_timer_data *rte_timer_data_arr;static const uint32_t default_data_id;static uint32_t rte_timer_subsystem_initialized;@@ -157,28 +160,30 @@ rte_timer_subsystem_init_v1905(void)int i, lcore_id;static const char *mz_name ="rte_timer_mz" ;const size_t data_arr_size = -RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +RTE_MAX_DATA_ELS *sizeof (*rte_timer_data_arr); +const size_t mem_size = data_arr_size +sizeof (*rte_timer_mz_refcnt);bool do_full_init = true;if (rte_timer_subsystem_initialized)return -EALREADY ; -reserve: - rte_errno =0 ; - mz =rte_memzone_reserve_aligned (mz_name, data_arr_size,SOCKET_ID_ANY , -0 ,RTE_CACHE_LINE_SIZE ); +rte_mcfg_timer_lock (); + + mz =rte_memzone_lookup (mz_name);if (mz ==NULL ) { -if (rte_errno ==EEXIST ) { - mz =rte_memzone_lookup (mz_name); -if (mz ==NULL ) - goto reserve; - - do_full_init = false; - }else + mz =rte_memzone_reserve_aligned (mz_name, mem_size, +SOCKET_ID_ANY ,0 ,RTE_CACHE_LINE_SIZE ); +if (mz ==NULL ) { +rte_mcfg_timer_unlock ();return -ENOMEM ; - } + } + do_full_init = true; + }else + do_full_init = false;Quoted 3 lines
lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void)
Quoted 3 lines
lib/librte_timer/rte_timer.c +2
-0+ rte_timer_data_mz = mz; rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr +
data_arr_size);Quoted 7 lines
lib/librte_timer/rte_timer.c rte_timer_subsystem_init_v1905(void) · line 202
Quoted 7 lines
if (do_full_init) { for (i = 0; i < RTE_MAX_DATA_ELS; i++) {lib/librte_timer/rte_timer.c +0
-0@@ -195,6 +200,9 @@ rte_timer_subsystem_init_v1905(void) } rte_timer_data_arr[default_data_id].internal_flags |=
FL_ALLOCATED;Quoted 20 lines
Quoted 20 lines
lib/librte_timer/rte_timer.c +10
-0+ (*rte_timer_mz_refcnt)++; + +rte_mcfg_timer_unlock (); rte_timer_subsystem_initialized =1 ;@@ -210,6 +218,13 @@ rte_timer_subsystem_finalize(void)if (!rte_timer_subsystem_initialized)return ; +rte_mcfg_timer_lock (); + +if (--(*rte_timer_mz_refcnt) ==0 ) +rte_memzone_free (rte_timer_data_mz); + +rte_mcfg_timer_unlock (); + rte_timer_subsystem_initialized =0 ; }Quoted 16 lines
Quoted 11 lines
lib/librte_timer/rte_timer.h +2
-0@@ -170,10 +170,11 @@ int __rte_experimental rte_timer_data_dealloc(uint32_t id); * Initializes internal variables (list, locks and so on) for the RTE * timer library. * + * @note + * This function must be called in every process before using theQuoted 5 lines
library.
+ *
* @return
* - 0: Success
- * --EEXIST: Returned in secondary process when primary process hasnot
Quoted 8 lines
Quoted 7 lines
- * yet initialized the timer subsystem
* --ENOMEM: Unable to allocate memory needed to initialize timer
* subsystem
*/
--
2.17.1
Quoted 7 lines
Quoted 2 lines
Can be squashed with the first patch.
This is not just search-and-replace - this is also a bugfix. I can squash the search-and-replace part with the first patch, but this commit will have to stay IMO.
Yes this is not search and replace, but the first patch is there for the bugfix.
There are no other uses of the newly introduced API, so the whole is a single change to me.
Apart from that, I am ok with the changes, you can add my review tag.
Signature · 1 line
David Marchand
v2 cover Fix timer resource leak
In reply to Anatoly BurakovPrevious attempts [1] at fixing the resource leak have been deemed unsuccessful because of limitations around what can be done without breaking the ABI. Now that we've broken the EAL ABI, we can fix this issue properly.
This patchset is adding a new lock API, as well as fixes the actual issue.
The patchset depends on mem config patchset [2].
[1] http://patches.dpdk.org/patch/53334/
[2] http://patches.dpdk.org/project/dpdk/list/?series=5369
Anatoly Burakov (1):
timer: fix resource leak in finalize
lib/librte_eal/common/eal_common_mcfg.c | 29 +++++++++++++ lib/librte_eal/common/eal_memcfg.h | 8 ++++ .../common/include/rte_eal_memconfig.h | 22 ++++++++++ lib/librte_eal/freebsd/eal/eal.c | 5 ++- lib/librte_eal/linux/eal/eal.c | 5 ++- lib/librte_eal/rte_eal_version.map | 2 + lib/librte_timer/rte_timer.c | 41 +++++++++++++------ lib/librte_timer/rte_timer.h | 5 ++- 8 files changed, 100 insertions(+), 17 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/1 timer: fix resource leak in finalize
Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions [1] because the data itself must live in shared memory.
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races, and implement the synchronization mechanism based on this lock in the timer library.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html
--- lib/librte_eal/common/eal_common_mcfg.c | 29 +++++++++++++ lib/librte_eal/common/eal_memcfg.h | 8 ++++ .../common/include/rte_eal_memconfig.h | 22 ++++++++++ lib/librte_eal/freebsd/eal/eal.c | 5 ++- lib/librte_eal/linux/eal/eal.c | 5 ++- lib/librte_eal/rte_eal_version.map | 2 + lib/librte_timer/rte_timer.c | 41 +++++++++++++------ lib/librte_timer/rte_timer.h | 5 ++- 8 files changed, 100 insertions(+), 17 deletions(-)
lib/librte_eal/common/eal_common_mcfg.c +29 -0
@@ -4,6 +4,7 @@#include <rte_config.h> #include <rte_eal_memconfig.h> +#include <rte_version.h> #include "eal_internal_cfg.h" #include "eal_memcfg.h"
@@ -31,6 +32,18 @@ eal_mcfg_wait_complete(void)rte_pause (); } +int +eal_mcfg_check_version (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; + +/* check if version from memconfig matches compiled in macro */ +if (mcfg->version !=RTE_VERSION ) +return -1 ; + +return 0 ; +} +void eal_mcfg_update_internal (void ) {
@@ -47,6 +60,8 @@ eal_mcfg_update_from_internal(void) mcfg->legacy_mem = internal_config.legacy_mem; mcfg->single_file_segments = internal_config.single_file_segments; +/* record current DPDK version */ + mcfg->version =RTE_VERSION ; }void
@@ -132,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config;rte_rwlock_write_unlock (&mcfg->mplock); } + +void +rte_mcfg_timer_lock (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +rte_spinlock_lock (&mcfg->tlock); +} + +void +rte_mcfg_timer_unlock (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +rte_spinlock_unlock (&mcfg->tlock); +}
lib/librte_eal/common/eal_memcfg.h +8 -0
@@ -11,6 +11,7 @@#include <rte_memory.h> #include <rte_memzone.h> #include <rte_pause.h> +#include <rte_spinlock.h> #include <rte_rwlock.h> #include <rte_tailq.h>
@@ -19,6 +20,8 @@*/ struct rte_mem_config {volatile uint32_t magic;/**< Magic number - sanity check. */ +uint32_t version; +/**< Prevent secondary processes using different DPDK versions. */ /* memory topology */ uint32_t nchannel;/**< Number of channels (0 if unknown). */
@@ -34,6 +37,7 @@ struct rte_mem_config {rte_rwlock_t mlock;/**< used by memzones for thread safety. */ rte_rwlock_t qlock;/**< used by tailqs for thread safety. */ rte_rwlock_t mplock;/**< used by mempool library for thread safety. */ +rte_spinlock_t tlock;/**< used by timer library for thread safety. */ rte_rwlock_t memory_hotplug_lock;/**< Indicates whether memory hotplug request is in progress. */
@@ -81,6 +85,10 @@ eal_mcfg_update_from_internal(void);void eal_mcfg_wait_complete (void ); +/* check if DPDK version of current process matches one stored in the config */ +int +eal_mcfg_check_version (void ); +/* set mem config as complete */ void eal_mcfg_complete (void );
lib/librte_eal/common/include/rte_eal_memconfig.h +22 -0
@@ -5,6 +5,8 @@#ifndef _RTE_EAL_MEMCONFIG_H_#define _RTE_EAL_MEMCONFIG_H_ +#include <rte_compat.h> +/** * @file *
@@ -87,6 +89,26 @@ rte_mcfg_mempool_write_lock(void);void rte_mcfg_mempool_write_unlock (void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Lock the internal EAL Timer Library lock for exclusive access. +*/ +__rte_experimental +void +rte_mcfg_timer_lock (void ); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Unlock the internal EAL Timer Library lock for exclusive access. +*/ +__rte_experimental +void +rte_mcfg_timer_unlock (void ); + #ifdef __cplusplus }#endif
lib/librte_eal/freebsd/eal/eal.c +4 -1
@@ -385,6 +385,10 @@ rte_config_init(void)if (rte_eal_config_attach () <0 )return -1 ;eal_mcfg_wait_complete (); +if (eal_mcfg_check_version () <0 ) { +RTE_LOG (ERR ,EAL ,"Primary and secondary process DPDK version mismatch\n" ); +return -1 ; + }if (rte_eal_config_reattach () <0 )return -1 ;eal_mcfg_update_internal ();
@@ -395,7 +399,6 @@ rte_config_init(void) rte_config.process_type);return -1 ; } -return 0 ; }
lib/librte_eal/linux/eal/eal.c +4 -1
@@ -489,6 +489,10 @@ rte_config_init(void)if (rte_eal_config_attach () <0 )return -1 ;eal_mcfg_wait_complete (); +if (eal_mcfg_check_version () <0 ) { +RTE_LOG (ERR ,EAL ,"Primary and secondary process DPDK version mismatch\n" ); +return -1 ; + }if (rte_eal_config_reattach () <0 )return -1 ;eal_mcfg_update_internal ();
@@ -499,7 +503,6 @@ rte_config_init(void) rte_config.process_type);return -1 ; } -return 0 ; }
lib/librte_eal/rte_eal_version.map +2 -0
@@ -405,4 +405,6 @@ EXPERIMENTAL { # added in 19.08 rte_lcore_cpuset; rte_lcore_to_cpu_id; + rte_mcfg_timer_lock; + rte_mcfg_timer_unlock; };
lib/librte_timer/rte_timer.c +28 -13
80 lines not on this page · raw message
lib/librte_timer/rte_timer.h +3 -2
@@ -172,10 +172,11 @@ int rte_timer_data_dealloc(uint32_t id);* Initializes internal variables (list, locks and so on) for the RTE * timer library. * +* @note +* This function must be called in every process before using the library. +* * @return * - 0: Success -* - -EEXIST: Returned in secondary process when primary process has not -* yet initialized the timer subsystem * - -ENOMEM: Unable to allocate memory needed to initialize timer * subsystem */
Signature · 1 line
2.17.1
v3 cover Fix timer resource leak
In reply to Anatoly BurakovPrevious attempts [1] at fixing the resource leak have been deemed unsuccessful because of limitations around what can be done without breaking the ABI. Now that we've broken the EAL ABI, we can fix this issue properly.
This patchset is adding a new lock API, as well as fixes the actual issue.
The patchset depends on mem config patchset [2].
v3:
- Accidentally squashed with previous patch, reverted
v2:
- Addressed review comments and squashed patches
[1] http://patches.dpdk.org/patch/53334/
[2] http://patches.dpdk.org/project/dpdk/list/?series=5369
Anatoly Burakov (1):
timer: fix resource leak in finalize
lib/librte_eal/common/eal_common_mcfg.c | 14 +++++++ lib/librte_eal/common/eal_memcfg.h | 2 + .../common/include/rte_eal_memconfig.h | 22 ++++++++++ lib/librte_eal/rte_eal_version.map | 2 + lib/librte_timer/rte_timer.c | 41 +++++++++++++------ lib/librte_timer/rte_timer.h | 5 ++- 6 files changed, 71 insertions(+), 15 deletions(-)
Signature · 1 line
2.17.1
PATCH v3 1/1 timer: fix resource leak in finalize
Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions [1] because the data itself must live in shared memory.
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races, and implement the synchronization mechanism based on this lock in the timer library.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.html
--- lib/librte_eal/common/eal_common_mcfg.c | 14 +++++++ lib/librte_eal/common/eal_memcfg.h | 2 + .../common/include/rte_eal_memconfig.h | 22 ++++++++++ lib/librte_eal/rte_eal_version.map | 2 + lib/librte_timer/rte_timer.c | 41 +++++++++++++------ lib/librte_timer/rte_timer.h | 5 ++- 6 files changed, 71 insertions(+), 15 deletions(-)
lib/librte_eal/common/eal_common_mcfg.c +14 -0
@@ -147,3 +147,17 @@ rte_mcfg_mempool_write_unlock(void)struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config;rte_rwlock_write_unlock (&mcfg->mplock); } + +void +rte_mcfg_timer_lock (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +rte_spinlock_lock (&mcfg->tlock); +} + +void +rte_mcfg_timer_unlock (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +rte_spinlock_unlock (&mcfg->tlock); +}
lib/librte_eal/common/eal_memcfg.h +2 -0
@@ -11,6 +11,7 @@#include <rte_memory.h> #include <rte_memzone.h> #include <rte_pause.h> +#include <rte_spinlock.h> #include <rte_rwlock.h> #include <rte_tailq.h>
@@ -36,6 +37,7 @@ struct rte_mem_config {rte_rwlock_t mlock;/**< used by memzones for thread safety. */ rte_rwlock_t qlock;/**< used by tailqs for thread safety. */ rte_rwlock_t mplock;/**< used by mempool library for thread safety. */ +rte_spinlock_t tlock;/**< used by timer library for thread safety. */ rte_rwlock_t memory_hotplug_lock;/**< Indicates whether memory hotplug request is in progress. */
lib/librte_eal/common/include/rte_eal_memconfig.h +22 -0
@@ -5,6 +5,8 @@#ifndef _RTE_EAL_MEMCONFIG_H_#define _RTE_EAL_MEMCONFIG_H_ +#include <rte_compat.h> +/** * @file *
@@ -87,6 +89,26 @@ rte_mcfg_mempool_write_lock(void);void rte_mcfg_mempool_write_unlock (void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Lock the internal EAL Timer Library lock for exclusive access. +*/ +__rte_experimental +void +rte_mcfg_timer_lock (void ); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Unlock the internal EAL Timer Library lock for exclusive access. +*/ +__rte_experimental +void +rte_mcfg_timer_unlock (void ); + #ifdef __cplusplus }#endif
lib/librte_eal/rte_eal_version.map +2 -0
@@ -405,4 +405,6 @@ EXPERIMENTAL { # added in 19.08 rte_lcore_cpuset; rte_lcore_to_cpu_id; + rte_mcfg_timer_lock; + rte_mcfg_timer_unlock; };
lib/librte_timer/rte_timer.c +28 -13
80 lines not on this page · raw message
lib/librte_timer/rte_timer.h +3 -2
@@ -172,10 +172,11 @@ int rte_timer_data_dealloc(uint32_t id);* Initializes internal variables (list, locks and so on) for the RTE * timer library. * +* @note +* This function must be called in every process before using the library. +* * @return * - 0: Success -* - -EEXIST: Returned in secondary process when primary process has not -* yet initialized the timer subsystem * - -ENOMEM: Unable to allocate memory needed to initialize timer * subsystem */
Signature · 1 line
2.17.1
Thomas Monjalon
Applied05/07/2019 19:22, Anatoly Burakov:
Quoted 17 lines
Currently, whenever timer library is initialized, the memory is leaked because there is no telling when primary or secondary processes get to use the state, and there is no way to initialize/deinitialize timer library state without race conditions [1] because the data itself must live in shared memory.
Add a spinlock to the shared mem config to have a way to exclusively initialize/deinitialize the timer library without any races, and implement the synchronization mechanism based on this lock in the timer library.
Also, update the API doc. Note that the behavior of the API itself did not change - the requirement to call init in every process was simply not documented explicitly.
Fixes c0749f7096c7 ("timer: allow management in shared memory")
Quoted 7 lines
[1] See the following email thread:
https://mails.dpdk.org/archives/dev/2019-May/131498.htmlSigned-off-by Erik Gabriel Carrillo <[email protected]>Signed-off-by Anatoly Burakov <[email protected]>Reviewed-by David Marchand <[email protected]>
Applied, thanks