Skip to contentSkip to conversation

eal: make lcore_config private

PATCH Superseded Stephen Hemminger 2 participants 6 messages Sep 25, 2019 – Oct 22, 2019

lore
Share & tools
lore
#0 lore raw

PATCH v7 1/1 eal: make lcore_config private

The internal structure of lcore_config is no longer be part of visible API/ABI. Make it private to EAL.

Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).

---
v7 - add eal_private.h to windows

 lib/librte_eal/common/eal_common_launch.c |  2 ++
 lib/librte_eal/common/eal_private.h       | 22 +++++++++++++++++++++
 lib/librte_eal/common/include/rte_lcore.h | 24 -----------------------
 lib/librte_eal/common/rte_service.c       |  2 ++
 lib/librte_eal/rte_eal_version.map        |  1 -
 lib/librte_eal/windows/eal/eal_thread.c   |  1 +
 6 files changed, 27 insertions(+), 25 deletions(-)
lib/librte_eal/common/eal_common_launch.c +2 -0
index fe0ba3f0d617..cf52d717f68e 100644
@@ -15,6 +15,8 @@
 #include <rte_per_lcore.h>
 #include <rte_lcore.h>
 
+#include "eal_private.h"
+
 /*
  * Wait until a lcore finished its job.
  */
lib/librte_eal/common/eal_private.h +22 -0
index 798ede553b21..25e80547904f 100644
@@ -10,6 +10,28 @@
 #include <stdio.h>
 
 #include <rte_dev.h>
+#include <rte_lcore.h>
+
+/**
+ * Structure storing internal configuration (per-lcore)
+ */
+struct lcore_config {
+	uint32_t core_id;      /**< core number on socket for this lcore */
+	uint32_t core_index;   /**< relative index, starting from 0 */
+	uint16_t socket_id;    /**< physical socket id for this lcore */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+	uint8_t detected;          /**< true if lcore was detected */
+	volatile enum rte_lcore_state_t state; /**< lcore state */
+	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	pthread_t thread_id;       /**< pthread identifier */
+	int pipe_master2slave[2];  /**< communication pipe with master */
+	int pipe_slave2master[2];  /**< communication pipe with master */
+	lcore_function_t * volatile f;         /**< function to call */
+	void * volatile arg;       /**< argument of function */
+	volatile int ret;          /**< return value of function */
+};
+
+extern struct lcore_config lcore_config[RTE_MAX_LCORE];
 
 /**
  * Initialize the memzone subsystem (private to eal).
lib/librte_eal/common/include/rte_lcore.h +0 -24
index c86f72eb12a8..0c683919564e 100644
@@ -66,30 +66,6 @@ typedef cpuset_t rte_cpuset_t;
 } while (0)
 #endif
 
-/**
- * Structure storing internal configuration (per-lcore)
- */
-struct lcore_config {
-	unsigned detected;         /**< true if lcore was detected */
-	pthread_t thread_id;       /**< pthread identifier */
-	int pipe_master2slave[2];  /**< communication pipe with master */
-	int pipe_slave2master[2];  /**< communication pipe with master */
-	lcore_function_t * volatile f;         /**< function to call */
-	void * volatile arg;       /**< argument of function */
-	volatile int ret;          /**< return value of function */
-	volatile enum rte_lcore_state_t state; /**< lcore state */
-	unsigned socket_id;        /**< physical socket id for this lcore */
-	unsigned core_id;          /**< core number on socket for this lcore */
-	int core_index;            /**< relative index, starting from 0 */
-	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
-	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
-};
-
-/**
- * Internal configuration (per-lcore)
- */
-extern struct lcore_config lcore_config[RTE_MAX_LCORE];
-
 RTE_DECLARE_PER_LCORE(unsigned, _lcore_id);  /**< Per thread "lcore id". */
 RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread "cpuset". */
 
lib/librte_eal/common/rte_service.c +2 -0
index c3653ebae46c..6e21f549051b 100644
@@ -21,6 +21,8 @@
 #include <rte_memory.h>
 #include <rte_malloc.h>
 
+#include "eal_private.h"
+
 #define RTE_SERVICE_NUM_MAX 64
 
 #define SERVICE_F_REGISTERED    (1 << 0)
lib/librte_eal/rte_eal_version.map +0 -1
index 7cbf82d37b0a..aeedf397764f 100644
@@ -4,7 +4,6 @@ DPDK_2.0 {
 	__rte_panic;
 	eal_parse_sysfs_value;
 	eal_timer_source;
-	lcore_config;
 	per_lcore__lcore_id;
 	per_lcore__rte_errno;
 	rte_calloc;
lib/librte_eal/windows/eal/eal_thread.c +1 -0
index 906502f90982..0591d4c7fb06 100644
@@ -12,6 +12,7 @@
 #include <rte_common.h>
 #include <eal_thread.h>
 
+#include "eal_private.h"
 
 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
 
Signature · 1 line
2.20.1
#1 lore raw

Thanks for working on this.

Quoted 6 lines · Stephen Hemminger

The internal structure of lcore_config is no longer be part of visible API/ABI. Make it private to EAL.

Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).

This patch is missing the release notes update.

Quoted 46 lines

---
v7 - add eal_private.h to windows

 lib/librte_eal/common/eal_common_launch.c |  2 ++
 lib/librte_eal/common/eal_private.h       | 22 +++++++++++++++++++++
 lib/librte_eal/common/include/rte_lcore.h | 24 -----------------------
 lib/librte_eal/common/rte_service.c       |  2 ++
 lib/librte_eal/rte_eal_version.map        |  1 -
 lib/librte_eal/windows/eal/eal_thread.c   |  1 +
 6 files changed, 27 insertions(+), 25 deletions(-)
lib/librte_eal/common/eal_common_launch.c +2 -0
index fe0ba3f0d617..cf52d717f68e 100644
@@ -15,6 +15,8 @@
 #include <rte_per_lcore.h>
 #include <rte_lcore.h>
 
+#include "eal_private.h"
+
 /*
  * Wait until a lcore finished its job.
  */
lib/librte_eal/common/eal_private.h +12 -0
index 798ede553b21..25e80547904f 100644
@@ -10,6 +10,28 @@
 #include <stdio.h>
 
 #include <rte_dev.h>
+#include <rte_lcore.h>
+
+/**
+ * Structure storing internal configuration (per-lcore)
+ */
+struct lcore_config {
+       uint32_t core_id;      /**< core number on socket for this lcore */
+       uint32_t core_index;   /**< relative index, starting from 0 */
+       uint16_t socket_id;    /**< physical socket id for this lcore */
+       uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+       uint8_t detected;          /**< true if lcore was detected */
+       volatile enum rte_lcore_state_t state; /**< lcore state */

lib/librte_eal/common/eal_private.h line 34

Quoted 10 lines
lib/librte_eal/common/eal_private.h +10 -0
+       rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+       pthread_t thread_id;       /**< pthread identifier */
+       int pipe_master2slave[2];  /**< communication pipe with master */
+       int pipe_slave2master[2];  /**< communication pipe with master */
+       lcore_function_t * volatile f;         /**< function to call */
+       void * volatile arg;       /**< argument of function */
+       volatile int ret;          /**< return value of function */
+};
+
+extern struct lcore_config lcore_config[RTE_MAX_LCORE];

Everything but cpuset can fit in a cache line.
You could just move the cpuset field at the end of the structure and change detected to uint8_t.
This gives the following layout:

struct lcore_config {

    pthread_t                  thread_id;            /*     0     8 */
    int                        pipe_master2slave[2]; /*     8     8 */
    int                        pipe_slave2master[2]; /*    16     8 */
    volatile lcore_function_t *  f;                  /*    24     8 */
    volatile void *            arg;                  /*    32     8 */
    volatile int               ret;                  /*    40     4 */
    volatile enum rte_lcore_state_t  state;          /*    44     4 */
    unsigned int               socket_id;            /*    48     4 */
    unsigned int               core_id;              /*    52     4 */
    int                        core_index;           /*    56     4 */
    uint8_t                    detected;             /*    60     1 */
    uint8_t                    core_role;            /*    61     1 */
    /* XXX 2 bytes hole, try to pack */

    /* --- cacheline 1 boundary (64 bytes) --- */
    rte_cpuset_t               cpuset;               /*    64   128 */
    /* --- cacheline 3 boundary (192 bytes) --- */

    /* size: 192, cachelines: 3, members: 13 */
    /* sum members: 190, holes: 1, sum holes: 2 */
};

The resulting structure is only two bytes bigger than your proposal and does not touch existing integer types (avoiding the risk of some integer conversion on socket_id for example).

Signature · 1 line
David Marchand
#2 lore raw

PATCH v8 1/1 eal: make lcore_config private

In reply to Stephen Hemminger

The internal structure of lcore_config is no longer be part of visible API/ABI. Make it private to EAL.

Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).

---
v8 - update release notes
rearrange to move cpuset to second cache line

 doc/guides/rel_notes/deprecation.rst      |  4 ----
 doc/guides/rel_notes/release_19_11.rst    |  2 ++
 lib/librte_eal/common/eal_common_launch.c |  2 ++
 lib/librte_eal/common/eal_private.h       | 24 +++++++++++++++++++++++
 lib/librte_eal/common/include/rte_lcore.h | 24 -----------------------
 lib/librte_eal/common/rte_service.c       |  2 ++
 lib/librte_eal/rte_eal_version.map        |  1 -
 lib/librte_eal/windows/eal/eal_thread.c   |  1 +
 8 files changed, 31 insertions(+), 29 deletions(-)
doc/guides/rel_notes/deprecation.rst +0 -4
index 0ee8533b133c..89811a622591 100644
@@ -23,10 +23,6 @@ Deprecation Notices
 * eal: The function ``rte_eal_remote_launch`` will return new error codes
   after read or write error on the pipe, instead of calling ``rte_panic``.
 
-* eal: The ``lcore_config`` struct and global symbol will be made private to
-  remove it from the externally visible ABI and allow it to be updated in the
-  future.
-
 * eal: both declaring and identifying devices will be streamlined in v18.11.
   New functions will appear to query a specific port from buses, classes of
   device and device drivers. Device declaration will be made coherent with the
doc/guides/rel_notes/release_19_11.rst +2 -0
index 27cfbd9e38c6..e8d4f9d2ba28 100644
@@ -110,6 +110,8 @@ ABI Changes
    Also, make sure to start the actual text at the margin.
    =========================================================
 
+* eal: The ``lcore_config`` struct and global symbol are now private.
+
 
 Shared Library Versions
 -----------------------
lib/librte_eal/common/eal_common_launch.c +2 -0
index fe0ba3f0d617..cf52d717f68e 100644
@@ -15,6 +15,8 @@
 #include <rte_per_lcore.h>
 #include <rte_lcore.h>
 
+#include "eal_private.h"
+
 /*
  * Wait until a lcore finished its job.
  */
lib/librte_eal/common/eal_private.h +24 -0
index 798ede553b21..e63eaa5d262d 100644
@@ -10,6 +10,30 @@
 #include <stdio.h>
 
 #include <rte_dev.h>
+#include <rte_lcore.h>
+
+/**
+ * Structure storing internal configuration (per-lcore)
+ */
+struct lcore_config {
+	pthread_t thread_id;       /**< pthread identifier */
+	int pipe_master2slave[2];  /**< communication pipe with master */
+	int pipe_slave2master[2];  /**< communication pipe with master */
+
+	lcore_function_t * volatile f; /**< function to call */
+	void * volatile arg;       /**< argument of function */
+	volatile int ret;          /**< return value of function */
+
+	uint32_t core_id;          /**< core number on socket for this lcore */
+	uint32_t core_index;       /**< relative index, starting from 0 */
+	uint16_t socket_id;        /**< physical socket id for this lcore */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+	uint8_t detected;          /**< true if lcore was detected */
+	volatile enum rte_lcore_state_t state; /**< lcore state */
+	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+};
+
+extern struct lcore_config lcore_config[RTE_MAX_LCORE];
 
 /**
  * Initialize the memzone subsystem (private to eal).
lib/librte_eal/common/include/rte_lcore.h +0 -24
index c86f72eb12a8..0c683919564e 100644
@@ -66,30 +66,6 @@ typedef cpuset_t rte_cpuset_t;
 } while (0)
 #endif
 
-/**
- * Structure storing internal configuration (per-lcore)
- */
-struct lcore_config {
-	unsigned detected;         /**< true if lcore was detected */
-	pthread_t thread_id;       /**< pthread identifier */
-	int pipe_master2slave[2];  /**< communication pipe with master */
-	int pipe_slave2master[2];  /**< communication pipe with master */
-	lcore_function_t * volatile f;         /**< function to call */
-	void * volatile arg;       /**< argument of function */
-	volatile int ret;          /**< return value of function */
-	volatile enum rte_lcore_state_t state; /**< lcore state */
-	unsigned socket_id;        /**< physical socket id for this lcore */
-	unsigned core_id;          /**< core number on socket for this lcore */
-	int core_index;            /**< relative index, starting from 0 */
-	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
-	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
-};
-
-/**
- * Internal configuration (per-lcore)
- */
-extern struct lcore_config lcore_config[RTE_MAX_LCORE];
-
 RTE_DECLARE_PER_LCORE(unsigned, _lcore_id);  /**< Per thread "lcore id". */
 RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread "cpuset". */
 
lib/librte_eal/common/rte_service.c +2 -0
index c3653ebae46c..6e21f549051b 100644
@@ -21,6 +21,8 @@
 #include <rte_memory.h>
 #include <rte_malloc.h>
 
+#include "eal_private.h"
+
 #define RTE_SERVICE_NUM_MAX 64
 
 #define SERVICE_F_REGISTERED    (1 << 0)
lib/librte_eal/rte_eal_version.map +0 -1
index 7cbf82d37b0a..aeedf397764f 100644
@@ -4,7 +4,6 @@ DPDK_2.0 {
 	__rte_panic;
 	eal_parse_sysfs_value;
 	eal_timer_source;
-	lcore_config;
 	per_lcore__lcore_id;
 	per_lcore__rte_errno;
 	rte_calloc;
lib/librte_eal/windows/eal/eal_thread.c +1 -0
index 906502f90982..0591d4c7fb06 100644
@@ -12,6 +12,7 @@
 #include <rte_common.h>
 #include <eal_thread.h>
 
+#include "eal_private.h"
 
 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
 
Signature · 1 line
2.20.1
#3 lore raw
Quoted 7 lines · Stephen Hemminger
lib/librte_eal/common/eal_private.h +7 -0
+struct lcore_config {
+       pthread_t thread_id;       /**< pthread identifier */
+       int pipe_master2slave[2];  /**< communication pipe with master */
+       int pipe_slave2master[2];  /**< communication pipe with master */
+
+       lcore_function_t * volatile f; /**< function to call */
+       void * volatile arg;       /**< argument of function */

lib/librte_eal/common/eal_private.h line 34

Quoted 10 lines
lib/librte_eal/common/eal_private.h +10 -0
+       volatile int ret;          /**< return value of function */
+
+       uint32_t core_id;          /**< core number on socket for this lcore */
+       uint32_t core_index;       /**< relative index, starting from 0 */
+       uint16_t socket_id;        /**< physical socket id for this lcore */
+       uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+       uint8_t detected;          /**< true if lcore was detected */
+       volatile enum rte_lcore_state_t state; /**< lcore state */
+       rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+};

There are still changes on the core_id, core_index, socket_id that I am not confortable with (at this point).

I prepared a series for -rc1 on ABI changes in EAL (that I will send shortly). I took your patch without the changes on core_id, core_index and socket_id.

We can discuss those changes later, thanks.

Signature · 1 line
David Marchand
#4 lore raw
Quoted 23 lines · On Tue, 22 Oct 2019 11:05:01 +0200 David Marchand <[email protected]>…
Quoted 7 lines · Stephen Hemminger
lib/librte_eal/common/eal_private.h +7 -0
+struct lcore_config {
+       pthread_t thread_id;       /**< pthread identifier */
+       int pipe_master2slave[2];  /**< communication pipe with master */
+       int pipe_slave2master[2];  /**< communication pipe with master */
+
+       lcore_function_t * volatile f; /**< function to call */
+       void * volatile arg;       /**< argument of function */

lib/librte_eal/common/eal_private.h line 34

Quoted 10 lines
lib/librte_eal/common/eal_private.h +10 -0
+       volatile int ret;          /**< return value of function */
+
+       uint32_t core_id;          /**< core number on socket for this lcore */
+       uint32_t core_index;       /**< relative index, starting from 0 */
+       uint16_t socket_id;        /**< physical socket id for this lcore */
+       uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+       uint8_t detected;          /**< true if lcore was detected */
+       volatile enum rte_lcore_state_t state; /**< lcore state */
+       rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+};  

There are still changes on the core_id, core_index, socket_id that I am not confortable with (at this point).

Quoted 2 lines

I prepared a series for -rc1 on ABI changes in EAL (that I will send shortly). I took your patch without the changes on core_id, core_index and socket_id.

Why, please be more precise.

Do you expect to support more than 32 bit worth of cores?

#5 lore raw
Quoted 27 lines · Stephen Hemminger
Quoted 23 lines · On Tue, 22 Oct 2019 11:05:01 +0200 David Marchand <[email protected]>…
Quoted 7 lines · Stephen Hemminger
lib/librte_eal/common/eal_private.h +7 -0
+struct lcore_config {
+       pthread_t thread_id;       /**< pthread identifier */
+       int pipe_master2slave[2];  /**< communication pipe with master */
+       int pipe_slave2master[2];  /**< communication pipe with master */
+
+       lcore_function_t * volatile f; /**< function to call */
+       void * volatile arg;       /**< argument of function */

lib/librte_eal/common/eal_private.h line 34

Quoted 10 lines
lib/librte_eal/common/eal_private.h +10 -0
+       volatile int ret;          /**< return value of function */
+
+       uint32_t core_id;          /**< core number on socket for this lcore */
+       uint32_t core_index;       /**< relative index, starting from 0 */
+       uint16_t socket_id;        /**< physical socket id for this lcore */
+       uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+       uint8_t detected;          /**< true if lcore was detected */
+       volatile enum rte_lcore_state_t state; /**< lcore state */
+       rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+};

There are still changes on the core_id, core_index, socket_id that I am not confortable with (at this point).

Quoted 6 lines
Quoted 2 lines

I prepared a series for -rc1 on ABI changes in EAL (that I will send shortly). I took your patch without the changes on core_id, core_index and socket_id.

Why, please be more precise.

I commented earlier that there were integer conversion with the fields you changed.

core_id is ok, and a uint32_t would be fine, but this does not change the size. socket_id needs investigation, but should be safe.
I am nervous about core_index, because it is used as a signed integer.

It looks too dangerous to blindly accept this change with the only reason of saving space.

Signature · 1 line
David Marchand