Skip to contentSkip to conversation

make lcore_config less visible

RFC In review Stephen Hemminger 2 participants 8 messages Apr 3, 2019 – Apr 5, 2019

lore
Share & tools
lore
#0 lore raw

cover make lcore_config less visible

This is a compile tested only set of patches to make the current lcore_config structure internal to EAL.

The first 4 would go into current release, and the last would go into later release to make it fully internal.

Stephen Hemminger (5):
eal: add accessor functions for lcore_config
bus: use lcore accessor functions
examples/bond: use lcore accessor
app/test: use lcore accessor functions
eal: make lcore_config private

 app/test/test_cryptodev.c                 |  2 +-
 app/test/test_hash_readwrite_lf.c         | 12 ++--
 app/test/test_ring_perf.c                 | 22 +++---
 doc/guides/rel_notes/release_19_05.rst    |  7 ++
 drivers/bus/dpaa/dpaa_bus.c               |  6 +-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c  |  4 +-
 examples/bond/main.c                      |  4 +-
 lib/librte_eal/common/eal_common_launch.c |  2 +
 lib/librte_eal/common/eal_common_lcore.c  | 46 +++++++++++++
 lib/librte_eal/common/eal_private.h       | 22 ++++++
 lib/librte_eal/common/include/rte_lcore.h | 81 ++++++++++++-----------
 lib/librte_eal/common/rte_service.c       |  2 +
 lib/librte_eal/rte_eal_version.map        |  1 -
 13 files changed, 150 insertions(+), 61 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

RFC 1/5 eal: add accessor functions for lcore_config

The fields of the internal EAL core configuration are currently laid bare as part of the API. This is not good practice and limits fixing issues with layout and sizes.

Make new accessor functions for the fields used by current drivers and examples. Mark the state and return code functions as experimental since these values might change in the future and probably shouldn't have been used by non EAL code anyway.

---
 doc/guides/rel_notes/release_19_05.rst    |  7 +++
 lib/librte_eal/common/eal_common_lcore.c  | 46 ++++++++++++++++++
 lib/librte_eal/common/include/rte_lcore.h | 57 +++++++++++++++++------
 3 files changed, 96 insertions(+), 14 deletions(-)
doc/guides/rel_notes/release_19_05.rst +7 -0
index bdad1ddbeedf..995fbf177b93 100644
@@ -188,6 +188,13 @@ ABI Changes
   alignment for ``rte_crypto_asym_op`` to restore expected ``rte_crypto_op``
   layout and alignment.
 
+* eal: the lcore config structure ``struct lcore_config`` will be made
+  internal to the EAL in a future release. This will allow the structure to
+  change without impacting API or ABI. All accesses to fields of this
+  structure should be done by the corresponding accessor functions.
+  For example, instead of using ``lcore_config[lcore_id].socket_id``
+  the function ``rte_lcore_socket_id(lcore_id)`` should be used instead.
+
 
 Shared Library Versions
 -----------------------
lib/librte_eal/common/eal_common_lcore.c +46 -0

52 lines not on this page · raw message

lib/librte_eal/common/include/rte_lcore.h +43 -14

75 lines not on this page · raw message

Signature · 1 line
2.17.1
#2 lore raw

Re: [dpdk-dev] [RFC 1/5] eal: add accessor functions for lcore_config

Quoted 10 lines · Stephen Hemminger
lib/librte_eal/common/eal_common_lcore.c +1 -0
index 1cbac42286ba..806204d9f73d 100644
@@ -16,6 +16,52 @@
 #include "eal_private.h"
 #include "eal_thread.h"
 
+int rte_lcore_index(int lcore_id)

lib/librte_eal/common/eal_common_lcore.c line 29

Quoted 10 lines
lib/librte_eal/common/eal_common_lcore.c +9 -0
+{
+       if (unlikely(lcore_id >= RTE_MAX_LCORE))
+               return -1;
+
+       if (lcore_id < 0)
+               lcore_id = (int)rte_lcore_id();
+
+       return lcore_config[lcore_id].core_index;
+}
 

+

lib/librte_eal/common/eal_common_lcore.c

Quoted 13 lines
lib/librte_eal/common/eal_common_lcore.c +12 -0
+int rte_lcore_to_cpu_id(int lcore_id)
+{
+       if (unlikely(lcore_id >= RTE_MAX_LCORE))
+               return -1;
+
+       if (lcore_id < 0)
+               lcore_id = (int)rte_lcore_id();
+
+       return lcore_config[lcore_id].core_id;
+}
+
+rte_cpuset_t rte_lcore_cpuset(unsigned lcore_id)
 

unsigned int

+{

lib/librte_eal/common/eal_common_lcore.c

Quoted 3 lines
lib/librte_eal/common/eal_common_lcore.c +2 -0
+       return lcore_config[lcore_id].cpuset;
+}
 

I am a bit skeptical at what dpaa wants to do with this. Anyway, it can be used when we want to check the current cpuset.

+

lib/librte_eal/common/eal_common_lcore.c line 46

Quoted 2 lines
lib/librte_eal/common/eal_common_lcore.c +1 -0
+unsigned

unsigned int

+rte_lcore_to_socket_id(unsigned int lcore_id)

lib/librte_eal/common/eal_common_lcore.c line 53

Quoted 7 lines
lib/librte_eal/common/eal_common_lcore.c +6 -0
+{
+       return lcore_config[lcore_id].socket_id;
+}
+
+enum rte_lcore_state_t
+rte_lcore_state(unsigned lcore_id)

unsigned int

+{

lib/librte_eal/common/eal_common_lcore.c line 54

Quoted 3 lines
lib/librte_eal/common/eal_common_lcore.c +2 -0
+       return lcore_config[lcore_id].state;
+}

This is a duplicate for existing rte_eal_get_lcore_state() in lib/librte_eal/common/eal_common_launch.c.
So either we keep rte_eal_get_lcore_state() or we replace it with this new one.

+

lib/librte_eal/common/eal_common_lcore.c line 59

Quoted 3 lines
lib/librte_eal/common/eal_common_lcore.c +2 -0
+int
+rte_lcore_return_code(unsigned lcore_id)

unsigned int

+{

lib/librte_eal/common/eal_common_lcore.c line 62

Quoted 3 lines
lib/librte_eal/common/eal_common_lcore.c +2 -0
+       return lcore_config[lcore_id].ret;
+}

+

lib/librte_eal/common/eal_common_lcore.c line 59

Quoted 2 lines
lib/librte_eal/common/eal_common_lcore.c +1 -0
+

Double blank line.

static int

Quoted 3 lines

socket_id_cmp(const void *a, const void *b)
{

Signature · 1 line
David Marchand
#3 lore raw

Re: [dpdk-dev] [RFC 1/5] eal: add accessor functions for lcore_config

Quoted 40 lines · On Fri, 5 Apr 2019 13:01:10 +0200 David Marchand <[email protected]>…
Quoted 10 lines · Stephen Hemminger
lib/librte_eal/common/eal_common_lcore.c +1 -0
index 1cbac42286ba..806204d9f73d 100644
@@ -16,6 +16,52 @@
 #include "eal_private.h"
 #include "eal_thread.h"
 
+int rte_lcore_index(int lcore_id)

lib/librte_eal/common/eal_common_lcore.c line 29

Quoted 10 lines
lib/librte_eal/common/eal_common_lcore.c +9 -0
+{
+       if (unlikely(lcore_id >= RTE_MAX_LCORE))
+               return -1;
+
+       if (lcore_id < 0)
+               lcore_id = (int)rte_lcore_id();
+
+       return lcore_config[lcore_id].core_index;
+}
 

+

lib/librte_eal/common/eal_common_lcore.c

Quoted 13 lines
lib/librte_eal/common/eal_common_lcore.c +12 -0
+int rte_lcore_to_cpu_id(int lcore_id)
+{
+       if (unlikely(lcore_id >= RTE_MAX_LCORE))
+               return -1;
+
+       if (lcore_id < 0)
+               lcore_id = (int)rte_lcore_id();
+
+       return lcore_config[lcore_id].core_id;
+}
+
+rte_cpuset_t rte_lcore_cpuset(unsigned lcore_id)
 

unsigned int

Quoted 7 lines

+{

lib/librte_eal/common/eal_common_lcore.c

Quoted 3 lines
lib/librte_eal/common/eal_common_lcore.c +2 -0
+       return lcore_config[lcore_id].cpuset;
+}
 

I am a bit skeptical at what dpaa wants to do with this. Anyway, it can be used when we want to check the current cpuset.

Since cpuset is defined already, don't want to override it with unsigned.

Quoted 18 lines

+

lib/librte_eal/common/eal_common_lcore.c line 46

Quoted 2 lines
lib/librte_eal/common/eal_common_lcore.c +1 -0
+unsigned

unsigned int

+rte_lcore_to_socket_id(unsigned int lcore_id)

lib/librte_eal/common/eal_common_lcore.c line 53

Quoted 7 lines
lib/librte_eal/common/eal_common_lcore.c +6 -0
+{
+       return lcore_config[lcore_id].socket_id;
+}
+
+enum rte_lcore_state_t
+rte_lcore_state(unsigned lcore_id)

unsigned int

+{

Quoted 8 lines

lib/librte_eal/common/eal_common_lcore.c line 54

Quoted 3 lines
lib/librte_eal/common/eal_common_lcore.c +2 -0
+       return lcore_config[lcore_id].state;
+}

This is a duplicate for existing rte_eal_get_lcore_state() in lib/librte_eal/common/eal_common_launch.c.
So either we keep rte_eal_get_lcore_state() or we replace it with this new one.

Good point.

#4 lore raw

RFC 2/5 bus: use lcore accessor functions

In reply to Stephen Hemminger

The lcore_config structure will be hidden in future release.

---
 drivers/bus/dpaa/dpaa_bus.c              | 6 ++++--
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 4 +++-
 2 files changed, 7 insertions(+), 3 deletions(-)
drivers/bus/dpaa/dpaa_bus.c +4 -2
index ac20eccd53c1..08c822781d9d 100644
@@ -254,6 +254,7 @@ int rte_dpaa_portal_init(void *arg)
 	unsigned int cpu, lcore = rte_lcore_id();
 	int ret;
 	struct dpaa_portal *dpaa_io_portal;
+	rte_cpuset_t cpuset;
 
 	BUS_INIT_FUNC_TRACE();
 
@@ -263,12 +264,13 @@ int rte_dpaa_portal_init(void *arg)
 		if (lcore >= RTE_MAX_LCORE)
 			return -1;
 
-	cpu = lcore_config[lcore].core_id;
+	cpu = rte_lcore_to_cpu_id(lcore);
 
 	/* Set CPU affinity for this thread.*/
 	id = pthread_self();
+	cpuset = rte_lcore_cpuset(lcore);
 	ret = pthread_setaffinity_np(id, sizeof(cpu_set_t),
-			&lcore_config[lcore].cpuset);
+				     &cpuset);
 	if (ret) {
 		DPAA_BUS_LOG(ERR, "pthread_setaffinity_np failed on core :%u"
 			     " (lcore=%u) with ret: %d", cpu, lcore, ret);
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +3 -1
index 7bcbde840e63..8efb24af5c6a 100644
@@ -366,7 +366,9 @@ dpaa2_check_lcore_cpuset(void)
 
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		for (i = 0; i < RTE_MAX_LCORE; i++) {
-			if (CPU_ISSET(i, &lcore_config[lcore_id].cpuset)) {
+			rte_cpuset_t cpuset = rte_lcore_cpuset(lcore_id);
+
+			if (CPU_ISSET(i, &cpuset)) {
 				RTE_LOG(DEBUG, EAL, "lcore id = %u cpu=%u\n",
 					lcore_id, i);
 				if (dpaa2_cpu[lcore_id] != 0xffffffff) {
Signature · 1 line
2.17.1
#5 lore raw

RFC 3/5 examples/bond: use lcore accessor

In reply to Stephen Hemminger

Refering to lcore_config directly is no longer recommended.

---
 examples/bond/main.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
examples/bond/main.c +2 -2
index ef86194fff4a..48e6755ea294 100644
@@ -526,8 +526,8 @@ static void cmd_start_parsed(__attribute__((unused)) void *parsed_result,
 	int slave_core_id = rte_lcore_id();
 
 	rte_spinlock_trylock(&global_flag_stru_p->lock);
-	if (global_flag_stru_p->LcoreMainIsRunning == 0)	{
-		if (lcore_config[global_flag_stru_p->LcoreMainCore].state != WAIT)	{
+	if (global_flag_stru_p->LcoreMainIsRunning == 0) {
+		if (rte_lcore_state(global_flag_stru_p->LcoreMainCore) != WAIT) {
 			rte_spinlock_unlock(&global_flag_stru_p->lock);
 			return;
 		}
Signature · 1 line
2.17.1
#6 lore raw

RFC 4/5 app/test: use lcore accessor functions

In reply to Stephen Hemminger

Don't refer to lcore_config directly.

---
 app/test/test_cryptodev.c         |  2 +-
 app/test/test_hash_readwrite_lf.c | 12 ++++++------
 app/test/test_ring_perf.c         | 22 ++++++++++++----------
 3 files changed, 19 insertions(+), 17 deletions(-)
app/test/test_cryptodev.c +1 -1
index 2ff204137157..55c5ef7d97a9 100644
@@ -378,7 +378,7 @@ testsuite_setup(void)
 			strcpy(temp_str, vdev_args);
 			strlcat(temp_str, ";", sizeof(temp_str));
 			slave_core_count++;
-			socket_id = lcore_config[i].socket_id;
+			socket_id = rte_lcore_to_socket_id(i);
 		}
 		if (slave_core_count != 2) {
 			RTE_LOG(ERR, USER1,
app/test/test_hash_readwrite_lf.c +6 -6
index cbfd9322696b..4968d5b49dcd 100644
@@ -622,7 +622,7 @@ test_hash_add_no_ks_lookup_hit(struct rwc_perf *rwc_perf_results, int rwc_lf,
 			rte_eal_mp_wait_lcore();
 
 			for (i = 1; i <= rwc_core_cnt[n]; i++)
-				if (lcore_config[i].ret < 0)
+				if (rte_lcore_return_code(i) < 0)
 					goto err;
 
 			unsigned long long cycles_per_lookup =
@@ -694,7 +694,7 @@ test_hash_add_no_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf,
 			if (ret < 0)
 				goto err;
 			for (i = 1; i <= rwc_core_cnt[n]; i++)
-				if (lcore_config[i].ret < 0)
+				if (rte_lcore_return_code(i) < 0)
 					goto err;
 
 			unsigned long long cycles_per_lookup =
@@ -770,7 +770,7 @@ test_hash_add_ks_lookup_hit_non_sp(struct rwc_perf *rwc_perf_results,
 			if (ret < 0)
 				goto err;
 			for (i = 1; i <= rwc_core_cnt[n]; i++)
-				if (lcore_config[i].ret < 0)
+				if (rte_lcore_return_code(i) < 0)
 					goto err;
 
 			unsigned long long cycles_per_lookup =
@@ -846,7 +846,7 @@ test_hash_add_ks_lookup_hit_sp(struct rwc_perf *rwc_perf_results, int rwc_lf,
 			if (ret < 0)
 				goto err;
 			for (i = 1; i <= rwc_core_cnt[n]; i++)
-				if (lcore_config[i].ret < 0)
+				if (rte_lcore_return_code(i) < 0)
 					goto err;
 
 			unsigned long long cycles_per_lookup =
@@ -921,7 +921,7 @@ test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 			if (ret < 0)
 				goto err;
 			for (i = 1; i <= rwc_core_cnt[n]; i++)
-				if (lcore_config[i].ret < 0)
+				if (rte_lcore_return_code(i) < 0)
 					goto err;
 
 			unsigned long long cycles_per_lookup =
@@ -1022,7 +1022,7 @@ test_hash_multi_add_lookup(struct rwc_perf *rwc_perf_results, int rwc_lf,
 				rte_eal_mp_wait_lcore();
 
 				for (i = 1; i <= rwc_core_cnt[n]; i++)
-					if (lcore_config[i].ret < 0)
+					if (rte_lcore_return_code(i) < 0)
 						goto err;
 
 				unsigned long long cycles_per_lookup =
app/test/test_ring_perf.c +12 -10
index ebb3939f51d0..6eccccfe93b4 100644
@@ -52,10 +52,11 @@ get_two_hyperthreads(struct lcore_pair *lcp)
 		RTE_LCORE_FOREACH(id2) {
 			if (id1 == id2)
 				continue;
-			c1 = lcore_config[id1].core_id;
-			c2 = lcore_config[id2].core_id;
-			s1 = lcore_config[id1].socket_id;
-			s2 = lcore_config[id2].socket_id;
+
+			c1 = rte_lcore_to_cpu_id(id1);
+			c2 = rte_lcore_to_cpu_id(id2);
+			s1 = rte_lcore_to_socket_id(id1);
+			s2 = rte_lcore_to_socket_id(id2);
 			if ((c1 == c2) && (s1 == s2)){
 				lcp->c1 = id1;
 				lcp->c2 = id2;
@@ -75,10 +76,11 @@ get_two_cores(struct lcore_pair *lcp)
 		RTE_LCORE_FOREACH(id2) {
 			if (id1 == id2)
 				continue;
-			c1 = lcore_config[id1].core_id;
-			c2 = lcore_config[id2].core_id;
-			s1 = lcore_config[id1].socket_id;
-			s2 = lcore_config[id2].socket_id;
+
+			c1 = rte_lcore_to_cpu_id(id1);
+			c2 = rte_lcore_to_cpu_id(id2);
+			s1 = rte_lcore_to_socket_id(id1);
+			s2 = rte_lcore_to_socket_id(id2);
 			if ((c1 != c2) && (s1 == s2)){
 				lcp->c1 = id1;
 				lcp->c2 = id2;
@@ -98,8 +100,8 @@ get_two_sockets(struct lcore_pair *lcp)
 		RTE_LCORE_FOREACH(id2) {
 			if (id1 == id2)
 				continue;
-			s1 = lcore_config[id1].socket_id;
-			s2 = lcore_config[id2].socket_id;
+			s1 = rte_lcore_to_socket_id(id1);
+			s2 = rte_lcore_to_socket_id(id2);
 			if (s1 != s2){
 				lcp->c1 = id1;
 				lcp->c2 = id2;
Signature · 1 line
2.17.1
#7 lore raw

RFC 5/5 eal: make lcore_config private

In reply to Stephen Hemminger

The internal structure of lcore_config should not be part of visible API/ABI. Make it private to EAL.

Rearrange and resize the fields in the structure so it takes less memory (and cache footprint).

---
 lib/librte_eal/common/eal_common_launch.c |  2 ++
 lib/librte_eal/common/eal_private.h       | 22 +++++++++++++++++++++
 lib/librte_eal/common/include/rte_lcore.h | 24 -----------------------
 lib/librte_eal/common/rte_service.c       |  2 ++
 lib/librte_eal/rte_eal_version.map        |  1 -
 5 files changed, 26 insertions(+), 25 deletions(-)
lib/librte_eal/common/eal_common_launch.c +2 -0
index fe0ba3f0d617..cf52d717f68e 100644
@@ -15,6 +15,8 @@
 #include <rte_per_lcore.h>
 #include <rte_lcore.h>
 
+#include "eal_private.h"
+
 /*
  * Wait until a lcore finished its job.
  */
lib/librte_eal/common/eal_private.h +22 -0
index 798ede553b21..25e80547904f 100644
@@ -10,6 +10,28 @@
 #include <stdio.h>
 
 #include <rte_dev.h>
+#include <rte_lcore.h>
+
+/**
+ * Structure storing internal configuration (per-lcore)
+ */
+struct lcore_config {
+	uint32_t core_id;      /**< core number on socket for this lcore */
+	uint32_t core_index;   /**< relative index, starting from 0 */
+	uint16_t socket_id;    /**< physical socket id for this lcore */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
+	uint8_t detected;          /**< true if lcore was detected */
+	volatile enum rte_lcore_state_t state; /**< lcore state */
+	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	pthread_t thread_id;       /**< pthread identifier */
+	int pipe_master2slave[2];  /**< communication pipe with master */
+	int pipe_slave2master[2];  /**< communication pipe with master */
+	lcore_function_t * volatile f;         /**< function to call */
+	void * volatile arg;       /**< argument of function */
+	volatile int ret;          /**< return value of function */
+};
+
+extern struct lcore_config lcore_config[RTE_MAX_LCORE];
 
 /**
  * Initialize the memzone subsystem (private to eal).
lib/librte_eal/common/include/rte_lcore.h +0 -24
index 7477ed2d9550..0b157b3e07ce 100644
@@ -37,30 +37,6 @@ typedef cpuset_t rte_cpuset_t;
 } while (0)
 #endif
 
-/**
- * Structure storing internal configuration (per-lcore)
- */
-struct lcore_config {
-	unsigned detected;         /**< true if lcore was detected */
-	pthread_t thread_id;       /**< pthread identifier */
-	int pipe_master2slave[2];  /**< communication pipe with master */
-	int pipe_slave2master[2];  /**< communication pipe with master */
-	lcore_function_t * volatile f;         /**< function to call */
-	void * volatile arg;       /**< argument of function */
-	volatile int ret;          /**< return value of function */
-	volatile enum rte_lcore_state_t state; /**< lcore state */
-	unsigned socket_id;        /**< physical socket id for this lcore */
-	unsigned core_id;          /**< core number on socket for this lcore */
-	int core_index;            /**< relative index, starting from 0 */
-	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
-	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
-};
-
-/**
- * Internal configuration (per-lcore)
- */
-extern struct lcore_config lcore_config[RTE_MAX_LCORE];
-
 RTE_DECLARE_PER_LCORE(unsigned, _lcore_id);  /**< Per thread "lcore id". */
 RTE_DECLARE_PER_LCORE(rte_cpuset_t, _cpuset); /**< Per thread "cpuset". */
 
lib/librte_eal/common/rte_service.c +2 -0
index 5f75e5a53fbf..8d53d966446a 100644
@@ -21,6 +21,8 @@
 #include <rte_memory.h>
 #include <rte_malloc.h>
 
+#include "eal_private.h"
+
 #define RTE_SERVICE_NUM_MAX 64
 
 #define SERVICE_F_REGISTERED    (1 << 0)
lib/librte_eal/rte_eal_version.map +0 -1
index d6e375135ad1..291e56dabc91 100644
@@ -4,7 +4,6 @@ DPDK_2.0 {
 	__rte_panic;
 	eal_parse_sysfs_value;
 	eal_timer_source;
-	lcore_config;
 	per_lcore__lcore_id;
 	per_lcore__rte_errno;
 	rte_calloc;
Signature · 1 line
2.17.1