eal: make lcore_config private
PATCH v7 1/1 eal: make lcore_config private
The internal structure of lcore_config is no longer be part of visible API/ABI. Make it private to EAL.
Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).
---
v7 - add eal_private.h to windows
lib/librte_eal/common/eal_common_launch.c | 2 ++ lib/librte_eal/common/eal_private.h | 22 +++++++++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 24 ----------------------- lib/librte_eal/common/rte_service.c | 2 ++ lib/librte_eal/rte_eal_version.map | 1 - lib/librte_eal/windows/eal/eal_thread.c | 1 + 6 files changed, 27 insertions(+), 25 deletions(-)
lib/librte_eal/common/eal_common_launch.c +2 -0
@@ -15,6 +15,8 @@#include <rte_per_lcore.h> #include <rte_lcore.h> +#include "eal_private.h" +/* * Wait until a lcore finished its job. */
lib/librte_eal/common/eal_private.h +22 -0
@@ -10,6 +10,28 @@#include <stdio.h> #include <rte_dev.h> +#include <rte_lcore.h> + +/** +* Structure storing internal configuration (per-lcore) +*/ +struct lcore_config { +uint32_t core_id;/**< core number on socket for this lcore */ +uint32_t core_index;/**< relative index, starting from 0 */ +uint16_t socket_id;/**< physical socket id for this lcore */ +uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ +uint8_t detected;/**< true if lcore was detected */ +volatile enum rte_lcore_state_t state;/**< lcore state */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ +pthread_t thread_id;/**< pthread identifier */ +int pipe_master2slave[2 ];/**< communication pipe with master */ +int pipe_slave2master[2 ];/**< communication pipe with master */ +lcore_function_t *volatile f;/**< function to call */ +void *volatile arg;/**< argument of function */ +volatile int ret;/**< return value of function */ +}; + +extern struct lcore_config lcore_config[RTE_MAX_LCORE ];/** * Initialize the memzone subsystem (private to eal).
lib/librte_eal/common/include/rte_lcore.h +0 -24
@@ -66,30 +66,6 @@ typedef cpuset_t rte_cpuset_t; } while (0)#endif -/** -* Structure storing internal configuration (per-lcore) -*/ -struct lcore_config { -unsigned detected;/**< true if lcore was detected */ -pthread_t thread_id;/**< pthread identifier */ -int pipe_master2slave[2 ];/**< communication pipe with master */ -int pipe_slave2master[2 ];/**< communication pipe with master */ -lcore_function_t *volatile f;/**< function to call */ -void *volatile arg;/**< argument of function */ -volatile int ret;/**< return value of function */ -volatile enum rte_lcore_state_t state;/**< lcore state */ -unsigned socket_id;/**< physical socket id for this lcore */ -unsigned core_id;/**< core number on socket for this lcore */ -int core_index;/**< relative index, starting from 0 */ -rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ -uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ -}; - -/** -* Internal configuration (per-lcore) -*/ -extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */
lib/librte_eal/common/rte_service.c +2 -0
@@ -21,6 +21,8 @@#include <rte_memory.h> #include <rte_malloc.h> +#include "eal_private.h" +#define RTE_SERVICE_NUM_MAX 64#define SERVICE_F_REGISTERED (1 << 0)
lib/librte_eal/rte_eal_version.map +0 -1
@@ -4,7 +4,6 @@ DPDK_2.0 { __rte_panic; eal_parse_sysfs_value; eal_timer_source; - lcore_config; per_lcore__lcore_id; per_lcore__rte_errno; rte_calloc;
lib/librte_eal/windows/eal/eal_thread.c +1 -0
@@ -12,6 +12,7 @@#include <rte_common.h> #include <eal_thread.h> +#include "eal_private.h" RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
Signature · 1 line
2.20.1
Thanks for working on this.
Quoted 6 lines · Stephen Hemminger
The internal structure of
lcore_configis no longer be part of visible API/ABI. Make it private to EAL.Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).
This patch is missing the release notes update.
Quoted 46 lines
Signed-off-by Stephen Hemminger <[email protected]>---
v7 - addeal_private.hto windowslib/librte_eal/common/eal_common_launch.c | 2 ++ lib/librte_eal/common/eal_private.h | 22 +++++++++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 24 ----------------------- lib/librte_eal/common/rte_service.c | 2 ++ lib/librte_eal/rte_eal_version.map | 1 - lib/librte_eal/windows/eal/eal_thread.c | 1 + 6 files changed, 27 insertions(+), 25 deletions(-)lib/librte_eal/common/eal_common_launch.c +2
-0@@ -15,6 +15,8 @@#include <rte_per_lcore.h> #include <rte_lcore.h> +#include "eal_private.h" +/* * Wait until a lcore finished its job. */ lib/librte_eal/common/eal_private.h +12
-0@@ -10,6 +10,28 @@#include <stdio.h> #include <rte_dev.h> +#include <rte_lcore.h> + +/** +* Structure storing internal configuration (per-lcore) +*/ +struct lcore_config { +uint32_t core_id;/**< core number on socket for this lcore */ +uint32_t core_index;/**< relative index, starting from 0 */ +uint16_t socket_id;/**< physical socket id for this lcore */ +uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ +uint8_t detected;/**< true if lcore was detected */ +volatile enum rte_lcore_state_t state;/**< lcore state */
lib/librte_eal/common/eal_private.h line 34
Quoted 10 lines
lib/librte_eal/common/eal_private.h +10
-0+rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ +pthread_t thread_id;/**< pthread identifier */ +int pipe_master2slave[2 ];/**< communication pipe with master */ +int pipe_slave2master[2 ];/**< communication pipe with master */ +lcore_function_t *volatile f;/**< function to call */ +void *volatile arg;/**< argument of function */ +volatile int ret;/**< return value of function */ +}; + +extern struct lcore_config lcore_config[RTE_MAX_LCORE ];
Everything but cpuset can fit in a cache line.
You could just move the cpuset field at the end of the structure and change detected to uint8_t.
This gives the following layout:
struct lcore_config {
pthread_t thread_id; /* 0 8 */
int pipe_master2slave[2]; /* 8 8 */
int pipe_slave2master[2]; /* 16 8 */
volatile lcore_function_t * f; /* 24 8 */
volatile void * arg; /* 32 8 */
volatile int ret; /* 40 4 */
volatile enum rte_lcore_state_t state; /* 44 4 */
unsigned int socket_id; /* 48 4 */
unsigned int core_id; /* 52 4 */
int core_index; /* 56 4 */
uint8_t detected; /* 60 1 */
uint8_t core_role; /* 61 1 */ /* XXX 2 bytes hole, try to pack */
/* --- cacheline 1 boundary (64 bytes) --- */
rte_cpuset_t cpuset; /* 64 128 */
/* --- cacheline 3 boundary (192 bytes) --- */
/* size: 192, cachelines: 3, members: 13 */
/* sum members: 190, holes: 1, sum holes: 2 */
};The resulting structure is only two bytes bigger than your proposal and does not touch existing integer types (avoiding the risk of some integer conversion on socket_id for example).
Signature · 1 line
David Marchand
PATCH v8 1/1 eal: make lcore_config private
In reply to Stephen HemmingerThe internal structure of lcore_config is no longer be part of visible API/ABI. Make it private to EAL.
Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).
---
v8 - update release notes
rearrange to move cpuset to second cache line
doc/guides/rel_notes/deprecation.rst | 4 ---- doc/guides/rel_notes/release_19_11.rst | 2 ++ lib/librte_eal/common/eal_common_launch.c | 2 ++ lib/librte_eal/common/eal_private.h | 24 +++++++++++++++++++++++ lib/librte_eal/common/include/rte_lcore.h | 24 ----------------------- lib/librte_eal/common/rte_service.c | 2 ++ lib/librte_eal/rte_eal_version.map | 1 - lib/librte_eal/windows/eal/eal_thread.c | 1 + 8 files changed, 31 insertions(+), 29 deletions(-)
doc/guides/rel_notes/deprecation.rst +0 -4
@@ -23,10 +23,6 @@ Deprecation Notices * eal: The function ``rte_eal_remote_launch`` will return new error codes after read or write error on the pipe, instead of calling ``rte_panic``. -* eal: The ``lcore_config`` struct and global symbol will be made private to - remove it from the externally visible ABI and allow it to be updated in the - future. - * eal: both declaring and identifying devices will be streamlined in v18.11. New functions will appear to query a specific port from buses, classes of device and device drivers. Device declaration will be made coherent with the
doc/guides/rel_notes/release_19_11.rst +2 -0
@@ -110,6 +110,8 @@ ABI Changes Also, make sure to start the actual text at the margin. ========================================================= +* eal: The ``lcore_config`` struct and global symbol are now private. + Shared Library Versions -----------------------
lib/librte_eal/common/eal_common_launch.c +2 -0
@@ -15,6 +15,8 @@#include <rte_per_lcore.h> #include <rte_lcore.h> +#include "eal_private.h" +/* * Wait until a lcore finished its job. */
lib/librte_eal/common/eal_private.h +24 -0
@@ -10,6 +10,30 @@#include <stdio.h> #include <rte_dev.h> +#include <rte_lcore.h> + +/** +* Structure storing internal configuration (per-lcore) +*/ +struct lcore_config { +pthread_t thread_id;/**< pthread identifier */ +int pipe_master2slave[2 ];/**< communication pipe with master */ +int pipe_slave2master[2 ];/**< communication pipe with master */ + +lcore_function_t *volatile f;/**< function to call */ +void *volatile arg;/**< argument of function */ +volatile int ret;/**< return value of function */ + +uint32_t core_id;/**< core number on socket for this lcore */ +uint32_t core_index;/**< relative index, starting from 0 */ +uint16_t socket_id;/**< physical socket id for this lcore */ +uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ +uint8_t detected;/**< true if lcore was detected */ +volatile enum rte_lcore_state_t state;/**< lcore state */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ +}; + +extern struct lcore_config lcore_config[RTE_MAX_LCORE ];/** * Initialize the memzone subsystem (private to eal).
lib/librte_eal/common/include/rte_lcore.h +0 -24
@@ -66,30 +66,6 @@ typedef cpuset_t rte_cpuset_t; } while (0)#endif -/** -* Structure storing internal configuration (per-lcore) -*/ -struct lcore_config { -unsigned detected;/**< true if lcore was detected */ -pthread_t thread_id;/**< pthread identifier */ -int pipe_master2slave[2 ];/**< communication pipe with master */ -int pipe_slave2master[2 ];/**< communication pipe with master */ -lcore_function_t *volatile f;/**< function to call */ -void *volatile arg;/**< argument of function */ -volatile int ret;/**< return value of function */ -volatile enum rte_lcore_state_t state;/**< lcore state */ -unsigned socket_id;/**< physical socket id for this lcore */ -unsigned core_id;/**< core number on socket for this lcore */ -int core_index;/**< relative index, starting from 0 */ -rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ -uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ -}; - -/** -* Internal configuration (per-lcore) -*/ -extern struct lcore_config lcore_config[RTE_MAX_LCORE ]; -RTE_DECLARE_PER_LCORE (unsigned, _lcore_id);/**< Per thread "lcore id". */ RTE_DECLARE_PER_LCORE (rte_cpuset_t, _cpuset);/**< Per thread "cpuset". */
lib/librte_eal/common/rte_service.c +2 -0
@@ -21,6 +21,8 @@#include <rte_memory.h> #include <rte_malloc.h> +#include "eal_private.h" +#define RTE_SERVICE_NUM_MAX 64#define SERVICE_F_REGISTERED (1 << 0)
lib/librte_eal/rte_eal_version.map +0 -1
@@ -4,7 +4,6 @@ DPDK_2.0 { __rte_panic; eal_parse_sysfs_value; eal_timer_source; - lcore_config; per_lcore__lcore_id; per_lcore__rte_errno; rte_calloc;
lib/librte_eal/windows/eal/eal_thread.c +1 -0
@@ -12,6 +12,7 @@#include <rte_common.h> #include <eal_thread.h> +#include "eal_private.h" RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
Signature · 1 line
2.20.1
Quoted 7 lines · Stephen Hemminger
lib/librte_eal/common/eal_private.h +7
-0+struct lcore_config { +pthread_t thread_id;/**< pthread identifier */ +int pipe_master2slave[2 ];/**< communication pipe with master */ +int pipe_slave2master[2 ];/**< communication pipe with master */ + +lcore_function_t *volatile f;/**< function to call */ +void *volatile arg;/**< argument of function */
lib/librte_eal/common/eal_private.h line 34
Quoted 10 lines
lib/librte_eal/common/eal_private.h +10
-0+volatile int ret;/**< return value of function */ + +uint32_t core_id;/**< core number on socket for this lcore */ +uint32_t core_index;/**< relative index, starting from 0 */ +uint16_t socket_id;/**< physical socket id for this lcore */ +uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ +uint8_t detected;/**< true if lcore was detected */ +volatile enum rte_lcore_state_t state;/**< lcore state */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ +};
There are still changes on the core_id, core_index, socket_id that I am not confortable with (at this point).
I prepared a series for -rc1 on ABI changes in EAL (that I will send shortly). I took your patch without the changes on core_id, core_index and socket_id.
We can discuss those changes later, thanks.
Signature · 1 line
David Marchand
Quoted 23 lines · On Tue, 22 Oct 2019 11:05:01 +0200 David Marchand <[email protected]>…
Quoted 7 lines · Stephen Hemminger
lib/librte_eal/common/eal_private.h +7
-0+struct lcore_config { +pthread_t thread_id;/**< pthread identifier */ +int pipe_master2slave[2 ];/**< communication pipe with master */ +int pipe_slave2master[2 ];/**< communication pipe with master */ + +lcore_function_t *volatile f;/**< function to call */ +void *volatile arg;/**< argument of function */ lib/librte_eal/common/eal_private.h line 34
Quoted 10 lines
lib/librte_eal/common/eal_private.h +10
-0+volatile int ret;/**< return value of function */ + +uint32_t core_id;/**< core number on socket for this lcore */ +uint32_t core_index;/**< relative index, starting from 0 */ +uint16_t socket_id;/**< physical socket id for this lcore */ +uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ +uint8_t detected;/**< true if lcore was detected */ +volatile enum rte_lcore_state_t state;/**< lcore state */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ +};There are still changes on the
core_id,core_index,socket_idthat I am not confortable with (at this point).
Quoted 2 lines
I prepared a series for -rc1 on ABI changes in EAL (that I will send shortly). I took your patch without the changes on
core_id,core_indexandsocket_id.
Why, please be more precise.
Do you expect to support more than 32 bit worth of cores?
Quoted 27 lines · Stephen Hemminger
Quoted 23 lines · On Tue, 22 Oct 2019 11:05:01 +0200 David Marchand <[email protected]>…
Quoted 7 lines · Stephen Hemminger
lib/librte_eal/common/eal_private.h +7
-0+struct lcore_config { +pthread_t thread_id;/**< pthread identifier */ +int pipe_master2slave[2 ];/**< communication pipe with master */ +int pipe_slave2master[2 ];/**< communication pipe with master */ + +lcore_function_t *volatile f;/**< function to call */ +void *volatile arg;/**< argument of function */ lib/librte_eal/common/eal_private.h line 34
Quoted 10 lines
lib/librte_eal/common/eal_private.h +10
-0+volatile int ret;/**< return value of function */ + +uint32_t core_id;/**< core number on socket for this lcore */ +uint32_t core_index;/**< relative index, starting from 0 */ +uint16_t socket_id;/**< physical socket id for this lcore */ +uint8_t core_role;/**< role of core eg: OFF, RTE, SERVICE */ +uint8_t detected;/**< true if lcore was detected */ +volatile enum rte_lcore_state_t state;/**< lcore state */ +rte_cpuset_t cpuset;/**< cpu set which the lcore affinity to */ +};There are still changes on the
core_id,core_index,socket_idthat I am not confortable with (at this point).
Quoted 6 lines
Quoted 2 lines
I prepared a series for -rc1 on ABI changes in EAL (that I will send shortly). I took your patch without the changes on
core_id,core_indexandsocket_id.Why, please be more precise.
I commented earlier that there were integer conversion with the fields you changed.
core_id is ok, and a uint32_t would be fine, but this does not change the size. socket_id needs investigation, but should be safe.
I am nervous about core_index, because it is used as a signed integer.
It looks too dangerous to blindly accept this change with the only reason of saving space.
Signature · 1 line
David Marchand