Skip to contentSkip to conversation

eal: restrict ctrl threads to startup cpu affinity

PATCH In review David Marchand 4 participants 21 messages Feb 13, 2019 – Mar 7, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 eal: restrict ctrl threads to startup cpu affinity

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.

Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")
---
 lib/librte_eal/common/eal_common_options.c | 28 ++++++++++++++++++++++++++++
 lib/librte_eal/common/eal_common_thread.c  | 21 ++++-----------------
 lib/librte_eal/common/eal_internal_cfg.h   |  3 +++
 3 files changed, 35 insertions(+), 17 deletions(-)
lib/librte_eal/common/eal_common_options.c +28 -0
index 6c96f45..b766252 100644
@@ -217,6 +217,7 @@ struct device_option {
 	internal_cfg->create_uio_dev = 0;
 	internal_cfg->iova_mode = RTE_IOVA_DC;
 	internal_cfg->user_mbuf_pool_ops_name = NULL;
+	CPU_ZERO(&internal_cfg->ctrl_cpuset);
 	internal_cfg->init_complete = 0;
 }
 
@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c)
 	cfg->lcore_count -= removed;
 }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+	rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+	rte_cpuset_t default_set;
+	unsigned int lcore_id;
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (eal_cpu_detected(lcore_id) &&
+				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+			CPU_SET(lcore_id, cpuset);
+		}
+	}
+
+	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+				&default_set) < 0)
+		CPU_ZERO(&default_set);
+
+	CPU_AND(cpuset, cpuset, &default_set);
+
+	/* if no detected cpu is off, use master core */
+	if (!CPU_COUNT(cpuset))
+		CPU_SET(rte_get_master_lcore(), cpuset);
+}
+
 int
 eal_cleanup_config(struct internal_config *internal_cfg)
 {
@@ -1393,6 +1419,8 @@ static int xdigit2val(unsigned char c)
 		lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
 	}
 
+	compute_ctrl_threads_cpuset(internal_cfg);
+
 	/* if no memory amounts were requested, this will result in 0 and
 	 * will be overridden later, right after eal_hugepage_info_init() */
 	for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
lib/librte_eal/common/eal_common_thread.c +4 -17
index 48ef4d6..893b0c1 100644
@@ -16,6 +16,7 @@
 #include <rte_memory.h>
 #include <rte_log.h>
 
+#include "eal_internal_cfg.h"
 #include "eal_private.h"
 #include "eal_thread.h"
 
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)
 		const pthread_attr_t *attr,
 		void *(*start_routine)(void *), void *arg)
 {
+	rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;
 	struct rte_thread_ctrl_params *params;
-	unsigned int lcore_id;
-	rte_cpuset_t cpuset;
-	int cpu_found, ret;
+	int ret;
 
 	params = malloc(sizeof(*params));
 	if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg)
 				"Cannot set name for ctrl thread\n");
 	}
 
-	cpu_found = 0;
-	CPU_ZERO(&cpuset);
-	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		if (eal_cpu_detected(lcore_id) &&
-				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
-			CPU_SET(lcore_id, &cpuset);
-			cpu_found = 1;
-		}
-	}
-	/* if no detected cpu is off, use master core */
-	if (!cpu_found)
-		CPU_SET(rte_get_master_lcore(), &cpuset);
-
-	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
+	ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
 	if (ret < 0)
 		goto fail;
 
lib/librte_eal/common/eal_internal_cfg.h +3 -0
index 60eaead..edff09d 100644
@@ -13,6 +13,8 @@
 #include <rte_eal.h>
 #include <rte_pci_dev_feature_defs.h>
 
+#include "eal_thread.h"
+
 #define MAX_HUGEPAGE_SIZES 3  /**< support up to 3 page sizes */
 
 /*
@@ -73,6 +75,7 @@ struct internal_config {
 	unsigned num_hugepage_sizes;      /**< how many sizes on this system */
 	struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
 	enum rte_iova_mode iova_mode ;    /**< Set IOVA mode on this system  */
+	rte_cpuset_t ctrl_cpuset;         /**< cpuset for ctrl threads */
 	volatile unsigned int init_complete;
 	/**< indicates whether EAL has completed initialization */
 };
Signature · 1 line
1.8.3.1
#1 lore raw
Quoted 10 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.

Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity.

Need to fix this last sentence...

Quoted 28 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")
---
 lib/librte_eal/common/eal_common_options.c | 28
++++++++++++++++++++++++++++
 lib/librte_eal/common/eal_common_thread.c  | 21 ++++-----------------
 lib/librte_eal/common/eal_internal_cfg.h   |  3 +++
 3 files changed, 35 insertions(+), 17 deletions(-)
lib/librte_eal/common/eal_common_options.c +10 -0
index 6c96f45..b766252 100644
@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c)
        cfg->lcore_count -= removed;
 }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+       rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+       rte_cpuset_t default_set;
+       unsigned int lcore_id;
+
+       for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+               if (eal_cpu_detected(lcore_id) &&
+                               rte_lcore_has_role(lcore_id, ROLE_OFF)) {

lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1383

Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +9 -0
+                       CPU_SET(lcore_id, cpuset);
+               }
+       }
+
+       if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+                               &default_set) < 0)
+               CPU_ZERO(&default_set);
+
+       CPU_AND(cpuset, cpuset, &default_set);
 

CPU_AND is different on Freebsd.

*CPU*_*AND*(*cpuset*_*t* **dst*, *cpuset*_*t* **src*);

The *CPU*_*AND*() macro removes CPUs absent from *src* from
*dst*. (It is the

     *cpuset(9)*

<https://www.freebsd.org/cgi/man.cgi?query=cpuset&sektion=9&apropos=0&manpath=FreeBSD+11.0-RELEASE+and+Ports>
equivalent of the scalar: *dst* &= *src*.) *CPU*_*AND*_*ATOMIC*() is
similar, with the same atomic semantics as *CPU*_*OR*_*ATOMIC*().

Will fix in v2.

Signature · 1 line
David Marchand
#2 lore raw
In reply to David Marchand
Quoted 8 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.

Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

Quoted 5 lines

The cpuset is computed once at init before the original cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Hi David,

Maybe i didn't have enough coffee today and i'm missing something here, but how is this different? Removing the coremask cores from the cpuset will effectively "spawn the ctrl threads on anything that is not part of the EAL coremask" (which is "not that polite to the rest of the system"), unless the application was run with taskset.

Is "taskset" the key point here? I.e. by default, we're still "not polite", unless the user asks nicely? :)

Signature · 2 lines
Thanks,
Anatoly
#3 lore raw
Quoted 17 lines · Anatoly
Quoted 8 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.

Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

Quoted 5 lines

The cpuset is computed once at init before the original cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Hi David,

Quoted 9 lines

Maybe i didn't have enough coffee today and i'm missing something here, but how is this different? Removing the coremask cores from the cpuset will effectively "spawn the ctrl threads on anything that is not part of the EAL coremask" (which is "not that polite to the rest of the system"), unless the application was run with taskset.

Is "taskset" the key point here? I.e. by default, we're still "not polite", unless the user asks nicely? :)

Eheh, sorry, yes.
A bit more context then, if you want to clearly pin cpu resources for the processes on your system (let's say having virtual machines and a popular vswitch), I can only think of two solutions.
Either you have something to configure your processes to have them call sched_setaffinity/pthread_set_affinity_np, or you use taskset to get them "jailed" without them caring.

Before the incriminated commit, we were keeping all threads on the coremask that had been passed, but as Olivier said, we would end up with ctrl threads spanwed on core running dataplane threads as well.

Now, the ctrl threads can be spawned anywhere on all & ~coremask, with no way to configure this.
I considered adding a new eal option, but I think relying on the current cpu affinity is a better default behavior and I can't see drawbacks at the moment.

Signature · 1 line
David Marchand
#4 lore raw
Quoted 31 lines · David Marchand

On Thu, Feb 14, 2019 at 10:39 AM Burakov, Anatoly <[email protected] <mailto:[email protected]>> wrote:

On 13-Feb-19 4:13 PM, David Marchand wrote:
> Spawning the ctrl threads on anything that is not part of the eal > coremask is not that polite to the rest of the system.
>
> Rather than introduce yet another eal options for this, let's take > the startup cpu affinity as a reference and remove the eal coremask > from it.
> If no cpu is left, then we default to the master core.
>
> The cpuset is computed once at init before the original cpu affinity. >
> Fixes: d651ee4919cd ("eal: set affinity for control threads") > Signed-off-by: David Marchand <[email protected]
<mailto:[email protected]>>
> ---

Hi David,

Maybe i didn't have enough coffee today and i'm missing something here, but how is this different? Removing the coremask cores from the cpuset will effectively "spawn the ctrl threads on anything that is not part of
the EAL coremask" (which is "not that polite to the rest of the system"), unless the application was run with taskset.

Is "taskset" the key point here? I.e. by default, we're still "not polite", unless the user asks nicely? :)

Quoted 13 lines

Eheh, sorry, yes.
A bit more context then, if you want to clearly pin cpu resources for the processes on your system (let's say having virtual machines and a popular vswitch), I can only think of two solutions. Either you have something to configure your processes to have them call sched_setaffinity/pthread_set_affinity_np, or you use taskset to get them "jailed" without them caring.

Before the incriminated commit, we were keeping all threads on the coremask that had been passed, but as Olivier said, we would end up with ctrl threads spanwed on core running dataplane threads as well.

Now, the ctrl threads can be spawned anywhere on all & ~coremask, with no way to configure this.
I considered adding a new eal option, but I think relying on the current cpu affinity is a better default behavior and I can't see drawbacks at the moment.

Signature · 1 line
David Marchand

OK, that makes sense. However, i feel this behavior (both old and new, for that matter) should be better documented somewhere in the EAL docs.

Signature · 2 lines
Thanks,
Anatoly
#5 lore raw
Quoted 13 lines · Anatoly
Quoted 11 lines · David Marchand

A bit more context then, if you want to clearly pin cpu resources for the processes on your system (let's say having virtual machines and a popular vswitch), I can only think of two solutions.
Either you have something to configure your processes to have them call sched_setaffinity/pthread_set_affinity_np, or you use taskset to get them "jailed" without them caring.

Before the incriminated commit, we were keeping all threads on the coremask that had been passed, but as Olivier said, we would end up with ctrl threads spanwed on core running dataplane threads as well.

Quoted 10 lines
Quoted 5 lines

Now, the ctrl threads can be spawned anywhere on all & ~coremask, with no way to configure this.
I considered adding a new eal option, but I think relying on the current cpu affinity is a better default behavior and I can't see drawbacks at the moment.

OK, that makes sense. However, i feel this behavior (both old and new, for that matter) should be better documented somewhere in the EAL docs.

I'll see what I can add in the doc for v2.

Signature · 1 line
David Marchand
#6 lore raw
In reply to David Marchand
Quoted 36 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.

Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")
---
 lib/librte_eal/common/eal_common_options.c | 28
++++++++++++++++++++++++++++
 lib/librte_eal/common/eal_common_thread.c  | 21 ++++-----------------
 lib/librte_eal/common/eal_internal_cfg.h   |  3 +++
 3 files changed, 35 insertions(+), 17 deletions(-)
lib/librte_eal/common/eal_common_options.c +8 -0
index 6c96f45..b766252 100644
@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c)
        cfg->lcore_count -= removed;
 }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+       rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+       rte_cpuset_t default_set;
+       unsigned int lcore_id;
+
+       for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {

lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1381

Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +9 -0
+               if (eal_cpu_detected(lcore_id) &&
+                               rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+                       CPU_SET(lcore_id, cpuset);
+               }
+       }
+
+       if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+                               &default_set) < 0)
+               CPU_ZERO(&default_set);
 

Now that I am testing on Freebsd, I can see this block I took from the existing "auto detect" function is just wrong. pthread_(g|s)etaffinity_np return a > 0 error value when failing.

Signature · 1 line
David Marchand
#7 lore raw

PATCH v2 1/2 eal: fix potential incorrect pinning for ctrl threads

In reply to David Marchand

pthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")
---
 lib/librte_eal/common/eal_common_thread.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
lib/librte_eal/common/eal_common_thread.c +1 -1
index 48ef4d6..a3985ce 100644
@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)
 		CPU_SET(rte_get_master_lcore(), &cpuset);
 
 	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
-	if (ret < 0)
+	if (ret)
 		goto fail;
 
 	ret = pthread_barrier_wait(&params->configured);
Signature · 1 line
1.8.3.1
#8 lore raw

PATCH v2 2/2 eal: restrict ctrl threads to startup cpu affinity

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity is lost.

Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048
$ procstat -S 1057
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1057 100131 testpmd             -                     2    1 2
 1057 100140 testpmd             eal-intr-thread       1    1 0-1
 1057 100141 testpmd             rte_mp_handle         1    1 0-1
 1057 100142 testpmd             lcore-slave-3         3    1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048
$ procstat -S 1061
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1061 100131 testpmd             -                     2    2 2
 1061 100144 testpmd             eal-intr-thread       1    2 1
 1061 100145 testpmd             rte_mp_handle         1    2 1
 1061 100147 testpmd             lcore-slave-3         3    2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048
$ procstat -S 1065
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1065 100131 testpmd             -                     2    2 2
 1065 100148 testpmd             eal-intr-thread       2    2 2
 1065 100149 testpmd             rte_mp_handle         2    2 2
 1065 100150 testpmd             lcore-slave-3         3    2 3
Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Changes since v1:
- added some description in the prog guide
- fixed FreeBSD build

---
 doc/guides/prog_guide/env_abstraction_layer.rst | 14 +++++++++++++
 lib/librte_eal/common/eal_common_options.c      | 28 +++++++++++++++++++++++++
 lib/librte_eal/common/eal_common_thread.c       | 21 ++++---------------
 lib/librte_eal/common/eal_internal_cfg.h        |  3 +++
 lib/librte_eal/common/include/rte_lcore.h       | 17 +++++++++++----
 5 files changed, 62 insertions(+), 21 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +14 -0
index 929d76d..bfc66d9 100644
@@ -498,6 +498,20 @@ Those TLS include *_cpuset* and *_socket_id*:
 *	*_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY.
 
 
+Control Thread API
+~~~~~~~~~~~~~~~~~~
+
+It is possible to create Control Threads using the public API ``rte_ctrl_thread_create()``.
+Those threads can be used for management/infrastructure tasks and are used internally by DPDK for multi process support and interrupt handling.
+
+Those threads will be scheduled on cpus part of the original process cpu affinity from which the dataplane and service lcores are excluded.
+
+For example, on a 8 cpus system, starting a dpdk application with -l 2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD),
+
+- with no affinity configuration, the Control Threads will end up on 0-1,4-7 cpus.
+- with affinity restricted to 2-4, the Control Threads will end up on cpu 4.
+- with affinity restricted to 2-3, the Control Threads will end up on cpu 2 (master lcore, which is the default when no cpu is available).
+
 .. _known_issue_label:
 
 Known Issues
lib/librte_eal/common/eal_common_options.c +28 -0
index 1f45f82..fca3f83 100644
@@ -217,6 +217,7 @@ struct device_option {
 	internal_cfg->create_uio_dev = 0;
 	internal_cfg->iova_mode = RTE_IOVA_DC;
 	internal_cfg->user_mbuf_pool_ops_name = NULL;
+	CPU_ZERO(&internal_cfg->ctrl_cpuset);
 	internal_cfg->init_complete = 0;
 }
 
@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c)
 	cfg->lcore_count -= removed;
 }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+	rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+	rte_cpuset_t default_set;
+	unsigned int lcore_id;
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (eal_cpu_detected(lcore_id) &&
+				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+			CPU_SET(lcore_id, cpuset);
+		}
+	}
+
+	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+				&default_set) < 0)
+		CPU_ZERO(&default_set);
+
+	RTE_CPU_AND(cpuset, cpuset, &default_set);
+
+	/* if no detected cpu is off, use master core */
+	if (!CPU_COUNT(cpuset))
+		CPU_SET(rte_get_master_lcore(), cpuset);
+}
+
 int
 eal_cleanup_config(struct internal_config *internal_cfg)
 {
@@ -1392,6 +1418,8 @@ static int xdigit2val(unsigned char c)
 		lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
 	}
 
+	compute_ctrl_threads_cpuset(internal_cfg);
+
 	/* if no memory amounts were requested, this will result in 0 and
 	 * will be overridden later, right after eal_hugepage_info_init() */
 	for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
lib/librte_eal/common/eal_common_thread.c +4 -17
index a3985ce..14f206c 100644
@@ -16,6 +16,7 @@
 #include <rte_memory.h>
 #include <rte_log.h>
 
+#include "eal_internal_cfg.h"
 #include "eal_private.h"
 #include "eal_thread.h"
 
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)
 		const pthread_attr_t *attr,
 		void *(*start_routine)(void *), void *arg)
 {
+	rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;
 	struct rte_thread_ctrl_params *params;
-	unsigned int lcore_id;
-	rte_cpuset_t cpuset;
-	int cpu_found, ret;
+	int ret;
 
 	params = malloc(sizeof(*params));
 	if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg)
 				"Cannot set name for ctrl thread\n");
 	}
 
-	cpu_found = 0;
-	CPU_ZERO(&cpuset);
-	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		if (eal_cpu_detected(lcore_id) &&
-				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
-			CPU_SET(lcore_id, &cpuset);
-			cpu_found = 1;
-		}
-	}
-	/* if no detected cpu is off, use master core */
-	if (!cpu_found)
-		CPU_SET(rte_get_master_lcore(), &cpuset);
-
-	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
+	ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
 	if (ret)
 		goto fail;
 
lib/librte_eal/common/eal_internal_cfg.h +3 -0
index 60eaead..edff09d 100644
@@ -13,6 +13,8 @@
 #include <rte_eal.h>
 #include <rte_pci_dev_feature_defs.h>
 
+#include "eal_thread.h"
+
 #define MAX_HUGEPAGE_SIZES 3  /**< support up to 3 page sizes */
 
 /*
@@ -73,6 +75,7 @@ struct internal_config {
 	unsigned num_hugepage_sizes;      /**< how many sizes on this system */
 	struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
 	enum rte_iova_mode iova_mode ;    /**< Set IOVA mode on this system  */
+	rte_cpuset_t ctrl_cpuset;         /**< cpuset for ctrl threads */
 	volatile unsigned int init_complete;
 	/**< indicates whether EAL has completed initialization */
 };
lib/librte_eal/common/include/rte_lcore.h +13 -4
index 6e09d91..eb49132 100644
@@ -23,10 +23,18 @@
 #define LCORE_ID_ANY     UINT32_MAX       /**< Any lcore. */
 
 #if defined(__linux__)
-	typedef	cpu_set_t rte_cpuset_t;
+typedef	cpu_set_t rte_cpuset_t;
+#define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2)
 #elif defined(__FreeBSD__)
 #include <pthread_np.h>
-	typedef cpuset_t rte_cpuset_t;
+typedef cpuset_t rte_cpuset_t;
+#define RTE_CPU_AND(dst, src1, src2) do \
+{ \
+	cpuset_t tmp; \
+	CPU_COPY(src1, &tmp); \
+	CPU_AND(&tmp, src2); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
 #endif
 
 /**
@@ -280,8 +288,9 @@ struct lcore_config {
  * Create a control thread.
  *
  * Wrapper to pthread_create(), pthread_setname_np() and
- * pthread_setaffinity_np(). The dataplane and service lcores are
- * excluded from the affinity of the new thread.
+ * pthread_setaffinity_np(). The affinity of the new thread is based
+ * on the cpu affinity retrieved at the time rte_eal_init() was called,
+ * the dataplane and service lcores are then excluded.
  *
  * @param thread
  *   Filled with the thread id of the new created thread.
Signature · 1 line
1.8.3.1
#9 lore raw
Quoted 75 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity is lost.

Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1057
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1057 100131 testpmd             -                     2    1 2
  1057 100140 testpmd             eal-intr-thread       1    1 0-1
  1057 100141 testpmd             rte_mp_handle         1    1 0-1
  1057 100142 testpmd             lcore-slave-3         3    1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1061
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1061 100131 testpmd             -                     2    2 2
  1061 100144 testpmd             eal-intr-thread       1    2 1
  1061 100145 testpmd             rte_mp_handle         1    2 1
  1061 100147 testpmd             lcore-slave-3         3    2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1065
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1065 100131 testpmd             -                     2    2 2
  1065 100148 testpmd             eal-intr-thread       2    2 2
  1065 100149 testpmd             rte_mp_handle         2    2 2
  1065 100150 testpmd             lcore-slave-3         3    2 3
Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Changes since v1:
- added some description in the prog guide
- fixed FreeBSD build

---
  doc/guides/prog_guide/env_abstraction_layer.rst | 14 +++++++++++++
  lib/librte_eal/common/eal_common_options.c      | 28 +++++++++++++++++++++++++
  lib/librte_eal/common/eal_common_thread.c       | 21 ++++---------------
  lib/librte_eal/common/eal_internal_cfg.h        |  3 +++
  lib/librte_eal/common/include/rte_lcore.h       | 17 +++++++++++----
  5 files changed, 62 insertions(+), 21 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +3 -0
index 929d76d..bfc66d9 100644
@@ -498,6 +498,20 @@ Those TLS include *_cpuset* and *_socket_id*:
  *	*_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY.
 
 
+Control Thread API
+~~~~~~~~~~~~~~~~~~
+

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 513

Quoted 10 lines
doc/guides/prog_guide/env_abstraction_layer.rst +10 -0
+It is possible to create Control Threads using the public API ``rte_ctrl_thread_create()``.
+Those threads can be used for management/infrastructure tasks and are used internally by DPDK for multi process support and interrupt handling.
+
+Those threads will be scheduled on cpus part of the original process cpu affinity from which the dataplane and service lcores are excluded.
+
+For example, on a 8 cpus system, starting a dpdk application with -l 2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD),
+
+- with no affinity configuration, the Control Threads will end up on 0-1,4-7 cpus.
+- with affinity restricted to 2-4, the Control Threads will end up on cpu 4.
+- with affinity restricted to 2-3, the Control Threads will end up on cpu 2 (master lcore, which is the default when no cpu is available).

You're not winning anything by foregoing the 80 char limit on documentation (doxygen will still generate this correctly), but you're losing in readability when working in terminal. I would prefer if you didn't do those long lines :)

Thomas, do we want checkpatch to warn about this?

Quoted 26 lines
doc/guides/prog_guide/env_abstraction_layer.rst +1 -0
+
  .. _known_issue_label:
 
  Known Issues
lib/librte_eal/common/eal_common_options.c +7 -0
index 1f45f82..fca3f83 100644
@@ -217,6 +217,7 @@ struct device_option {
  	internal_cfg->create_uio_dev = 0;
  	internal_cfg->iova_mode = RTE_IOVA_DC;
  	internal_cfg->user_mbuf_pool_ops_name = NULL;
+	CPU_ZERO(&internal_cfg->ctrl_cpuset);
  	internal_cfg->init_complete = 0;
  }
 
@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c)
  	cfg->lcore_count -= removed;
  }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+	rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+	rte_cpuset_t default_set;
+	unsigned int lcore_id;

lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1378

Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +10 -0
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (eal_cpu_detected(lcore_id) &&
+				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+			CPU_SET(lcore_id, cpuset);
+		}
+	}
+
+	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+				&default_set) < 0)

Shouldn't this be != 0? Manpage doesn't say the error values will be negative.

Signature · 2 lines
Thanks,
Anatoly
#10 lore raw
Quoted 27 lines · Anatoly

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 498

Quoted 4 lines · David Marchand
doc/guides/prog_guide/env_abstraction_layer.rst +0 -0
@@ -498,6 +498,20 @@ Those TLS include *_cpuset* and *_socket_id*:
  *   *_socket_id* stores the NUMA node of the CPU set. If the CPUs in

CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY.

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*:

Quoted 6 lines
doc/guides/prog_guide/env_abstraction_layer.rst +4 -0
+Control Thread API
+~~~~~~~~~~~~~~~~~~
+
+It is possible to create Control Threads using the public API

``rte_ctrl_thread_create()``.

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 503

Quoted 1 line
doc/guides/prog_guide/env_abstraction_layer.rst +1 -0
+Those threads can be used for management/infrastructure tasks and are

used internally by DPDK for multi process support and interrupt handling.

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 505

Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2 -0
+
+Those threads will be scheduled on cpus part of the original process

cpu affinity from which the dataplane and service lcores are excluded.

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 507

Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2 -0
+
+For example, on a 8 cpus system, starting a dpdk application with -l

2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD),

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 509

Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2 -0
+
+- with no affinity configuration, the Control Threads will end up on

0-1,4-7 cpus.

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 510

Quoted 1 line
doc/guides/prog_guide/env_abstraction_layer.rst +1 -0
+- with affinity restricted to 2-4, the Control Threads will end up on
Quoted 9 lines

cpu 4.

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 511

Quoted 1 line
doc/guides/prog_guide/env_abstraction_layer.rst +1 -0
+- with affinity restricted to 2-3, the Control Threads will end up on

cpu 2 (master lcore, which is the default when no cpu is available).

You're not winning anything by foregoing the 80 char limit on documentation (doxygen will still generate this correctly), but you're losing in readability when working in terminal. I would prefer if you didn't do those long lines :)

I don't really care, I will just wait for Thomas opinion.

Quoted 34 lines

Thomas, do we want checkpatch to warn about this?

Review

Quoted 5 lines
doc/guides/prog_guide/env_abstraction_layer.rst +1 -0
+
  .. _known_issue_label:
 
  Known Issues
(quoted diff) +0 -0

b/lib/librte_eal/common/eal_common_options.c

Quoted 26 lines

index 1f45f82..fca3f83 100644

lib/librte_eal/common/eal_common_options.c +12 -0
@@ -217,6 +217,7 @@ struct device_option {
      internal_cfg->create_uio_dev = 0;
      internal_cfg->iova_mode = RTE_IOVA_DC;
      internal_cfg->user_mbuf_pool_ops_name = NULL;
+     CPU_ZERO(&internal_cfg->ctrl_cpuset);
      internal_cfg->init_complete = 0;
  }
 
@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c)
      cfg->lcore_count -= removed;
  }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+     rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+     rte_cpuset_t default_set;
+     unsigned int lcore_id;
+
+     for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+             if (eal_cpu_detected(lcore_id) &&
+                             rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+                     CPU_SET(lcore_id, cpuset);
Quoted 9 lines

lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1378

Quoted 5 lines
lib/librte_eal/common/eal_common_options.c +5 -0
+             }
+     }
+
+     if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+                             &default_set) < 0)

Shouldn't this be != 0? Manpage doesn't say the error values will be negative.

Good catch...
/me hides

Thanks for the review.

Signature · 1 line
David Marchand
#11 lore raw

19/02/2019 12:51, David Marchand:

Quoted 25 lines
Quoted 23 lines · Anatoly

doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*:

Quoted 6 lines · David Marchand
doc/guides/prog_guide/env_abstraction_layer.rst +4 -0
+Control Thread API
+~~~~~~~~~~~~~~~~~~
+
+It is possible to create Control Threads using the public API

``rte_ctrl_thread_create()``.

Quoted 1 line

+Those threads can be used for management/infrastructure tasks and are

used internally by DPDK for multi process support and interrupt handling.

doc/guides/prog_guide/env_abstraction_layer.rst

Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2 -0
+
+Those threads will be scheduled on cpus part of the original process

cpu affinity from which the dataplane and service lcores are excluded.

doc/guides/prog_guide/env_abstraction_layer.rst

Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2 -0
+
+For example, on a 8 cpus system, starting a dpdk application with -l

2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD),

Quoted 2 lines

+
+- with no affinity configuration, the Control Threads will end up on

0-1,4-7 cpus.

Quoted 1 line

+- with affinity restricted to 2-4, the Control Threads will end up on

cpu 4.

Quoted 1 line

+- with affinity restricted to 2-3, the Control Threads will end up on

Quoted 10 lines
Quoted 6 lines

cpu 2 (master lcore, which is the default when no cpu is available).

You're not winning anything by foregoing the 80 char limit on documentation (doxygen will still generate this correctly), but you're losing in readability when working in terminal. I would prefer if you didn't do those long lines :)

I don't really care, I will just wait for Thomas opinion.

Quoted 1 line

Thomas, do we want checkpatch to warn about this?

http://doc.dpdk.org/guides/contributing/documentation.html#line-length

Lines should not exceed 80 chars.
There is a tradition of being very flexible in docs.
The best is to wrap at the end of a logical group of words, like after commas or dots. So the doc updates patches are easier to read.

#12 lore raw

Re: [PATCH v2 1/2] eal: fix potential incorrect pinning for ctrl threads

In reply to David Marchand
Quoted 16 lines · David Marchand

pthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")
---
  lib/librte_eal/common/eal_common_thread.c | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
lib/librte_eal/common/eal_common_thread.c +0 -0
index 48ef4d6..a3985ce 100644

lib/librte_eal/common/eal_common_thread.c static void *rte_thread_init(void *arg) · line 215

Quoted 8 lines
lib/librte_eal/common/eal_common_thread.c +1 -1
@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)
  		CPU_SET(rte_get_master_lcore(), &cpuset);
 
  	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
-	if (ret < 0)
+	if (ret)
  		goto fail;
 
  	ret = pthread_barrier_wait(&params->configured);
CC stable?
Signature · 2 lines
Thanks,
Anatoly
#13 lore raw

Re: [PATCH v2 1/2] eal: fix potential incorrect pinning for ctrl threads

Quoted 24 lines · Anatoly

Review

Quoted 11 lines · David Marchand

pthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")
---
  lib/librte_eal/common/eal_common_thread.c | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
(quoted diff) +0 -0

b/lib/librte_eal/common/eal_common_thread.c

Quoted 11 lines

index 48ef4d6..a3985ce 100644

lib/librte_eal/common/eal_common_thread.c +1 -1
@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)
              CPU_SET(rte_get_master_lcore(), &cpuset);
 
      ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
-     if (ret < 0)
+     if (ret)
              goto fail;
 
Quoted 5 lines

lib/librte_eal/common/eal_common_thread.c static void *rte_thread_init(void *arg) · line 215

Quoted 2 lines
lib/librte_eal/common/eal_common_thread.c +0 -0
      ret = pthread_barrier_wait(&params->configured);
CC stable?

Yes, I again forgot to check with check-git-log.sh...

Signature · 1 line
David Marchand
#14 lore raw

PATCH v3 1/2 eal: fix potential incorrect pinning for ctrl threads

In reply to David Marchand

pthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")

---
Changelog since v2:
- added missing Cc: [email protected]

---
 lib/librte_eal/common/eal_common_thread.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
lib/librte_eal/common/eal_common_thread.c +1 -1
index 48ef4d6..a3985ce 100644
@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)
 		CPU_SET(rte_get_master_lcore(), &cpuset);
 
 	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
-	if (ret < 0)
+	if (ret)
 		goto fail;
 
 	ret = pthread_barrier_wait(&params->configured);
Signature · 1 line
1.8.3.1
#15 lore raw

PATCH v3 2/2 eal: restrict ctrl threads to startup cpu affinity

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity is lost.

Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048
$ procstat -S 1057
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1057 100131 testpmd             -                     2    1 2
 1057 100140 testpmd             eal-intr-thread       1    1 0-1
 1057 100141 testpmd             rte_mp_handle         1    1 0-1
 1057 100142 testpmd             lcore-slave-3         3    1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048
$ procstat -S 1061
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1061 100131 testpmd             -                     2    2 2
 1061 100144 testpmd             eal-intr-thread       1    2 1
 1061 100145 testpmd             rte_mp_handle         1    2 1
 1061 100147 testpmd             lcore-slave-3         3    2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048
$ procstat -S 1065
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1065 100131 testpmd             -                     2    2 2
 1065 100148 testpmd             eal-intr-thread       2    2 2
 1065 100149 testpmd             rte_mp_handle         2    2 2
 1065 100150 testpmd             lcore-slave-3         3    2 3
Fixes d651ee4919cd ("eal: set affinity for control threads")

---
Changes since v2:
- added missing Cc: [email protected]
- re-indent doc to comply with 80 columns
- fix incorrect check on pthread_getaffinity_np

Changes since v1:
- added some description in the prog guide
- fixed FreeBSD build

---
 doc/guides/prog_guide/env_abstraction_layer.rst | 22 +++++++++++++++++++
 lib/librte_eal/common/eal_common_options.c      | 28 +++++++++++++++++++++++++
 lib/librte_eal/common/eal_common_thread.c       | 21 ++++---------------
 lib/librte_eal/common/eal_internal_cfg.h        |  3 +++
 lib/librte_eal/common/include/rte_lcore.h       | 17 +++++++++++----
 5 files changed, 70 insertions(+), 21 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +22 -0
index 929d76d..1fd6ece 100644
@@ -498,6 +498,28 @@ Those TLS include *_cpuset* and *_socket_id*:
 *	*_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY.
 
 
+Control Thread API
+~~~~~~~~~~~~~~~~~~
+
+It is possible to create Control Threads using the public API
+``rte_ctrl_thread_create()``.
+Those threads can be used for management/infrastructure tasks and are used
+internally by DPDK for multi process support and interrupt handling.
+
+Those threads will be scheduled on cpus part of the original process cpu
+affinity from which the dataplane and service lcores are excluded.
+
+For example, on a 8 cpus system, starting a dpdk application with -l 2,3
+(dataplane cores), then depending on the affinity configuration which can be
+controlled with tools like taskset (Linux) or cpuset (FreeBSD),
+
+- with no affinity configuration, the Control Threads will end up on
+  0-1,4-7 cpus.
+- with affinity restricted to 2-4, the Control Threads will end up on
+  cpu 4.
+- with affinity restricted to 2-3, the Control Threads will end up on
+  cpu 2 (master lcore, which is the default when no cpu is available).
+
 .. _known_issue_label:
 
 Known Issues
lib/librte_eal/common/eal_common_options.c +28 -0
index 1f45f82..f8b7ab9 100644
@@ -217,6 +217,7 @@ struct device_option {
 	internal_cfg->create_uio_dev = 0;
 	internal_cfg->iova_mode = RTE_IOVA_DC;
 	internal_cfg->user_mbuf_pool_ops_name = NULL;
+	CPU_ZERO(&internal_cfg->ctrl_cpuset);
 	internal_cfg->init_complete = 0;
 }
 
@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c)
 	cfg->lcore_count -= removed;
 }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+	rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+	rte_cpuset_t default_set;
+	unsigned int lcore_id;
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (eal_cpu_detected(lcore_id) &&
+				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+			CPU_SET(lcore_id, cpuset);
+		}
+	}
+
+	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+				&default_set))
+		CPU_ZERO(&default_set);
+
+	RTE_CPU_AND(cpuset, cpuset, &default_set);
+
+	/* if no detected cpu is off, use master core */
+	if (!CPU_COUNT(cpuset))
+		CPU_SET(rte_get_master_lcore(), cpuset);
+}
+
 int
 eal_cleanup_config(struct internal_config *internal_cfg)
 {
@@ -1392,6 +1418,8 @@ static int xdigit2val(unsigned char c)
 		lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
 	}
 
+	compute_ctrl_threads_cpuset(internal_cfg);
+
 	/* if no memory amounts were requested, this will result in 0 and
 	 * will be overridden later, right after eal_hugepage_info_init() */
 	for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
lib/librte_eal/common/eal_common_thread.c +4 -17
index a3985ce..14f206c 100644
@@ -16,6 +16,7 @@
 #include <rte_memory.h>
 #include <rte_log.h>
 
+#include "eal_internal_cfg.h"
 #include "eal_private.h"
 #include "eal_thread.h"
 
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)
 		const pthread_attr_t *attr,
 		void *(*start_routine)(void *), void *arg)
 {
+	rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;
 	struct rte_thread_ctrl_params *params;
-	unsigned int lcore_id;
-	rte_cpuset_t cpuset;
-	int cpu_found, ret;
+	int ret;
 
 	params = malloc(sizeof(*params));
 	if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg)
 				"Cannot set name for ctrl thread\n");
 	}
 
-	cpu_found = 0;
-	CPU_ZERO(&cpuset);
-	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		if (eal_cpu_detected(lcore_id) &&
-				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
-			CPU_SET(lcore_id, &cpuset);
-			cpu_found = 1;
-		}
-	}
-	/* if no detected cpu is off, use master core */
-	if (!cpu_found)
-		CPU_SET(rte_get_master_lcore(), &cpuset);
-
-	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
+	ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
 	if (ret)
 		goto fail;
 
lib/librte_eal/common/eal_internal_cfg.h +3 -0
index 60eaead..edff09d 100644
@@ -13,6 +13,8 @@
 #include <rte_eal.h>
 #include <rte_pci_dev_feature_defs.h>
 
+#include "eal_thread.h"
+
 #define MAX_HUGEPAGE_SIZES 3  /**< support up to 3 page sizes */
 
 /*
@@ -73,6 +75,7 @@ struct internal_config {
 	unsigned num_hugepage_sizes;      /**< how many sizes on this system */
 	struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
 	enum rte_iova_mode iova_mode ;    /**< Set IOVA mode on this system  */
+	rte_cpuset_t ctrl_cpuset;         /**< cpuset for ctrl threads */
 	volatile unsigned int init_complete;
 	/**< indicates whether EAL has completed initialization */
 };
lib/librte_eal/common/include/rte_lcore.h +13 -4
index 6e09d91..eb49132 100644
@@ -23,10 +23,18 @@
 #define LCORE_ID_ANY     UINT32_MAX       /**< Any lcore. */
 
 #if defined(__linux__)
-	typedef	cpu_set_t rte_cpuset_t;
+typedef	cpu_set_t rte_cpuset_t;
+#define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2)
 #elif defined(__FreeBSD__)
 #include <pthread_np.h>
-	typedef cpuset_t rte_cpuset_t;
+typedef cpuset_t rte_cpuset_t;
+#define RTE_CPU_AND(dst, src1, src2) do \
+{ \
+	cpuset_t tmp; \
+	CPU_COPY(src1, &tmp); \
+	CPU_AND(&tmp, src2); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
 #endif
 
 /**
@@ -280,8 +288,9 @@ struct lcore_config {
  * Create a control thread.
  *
  * Wrapper to pthread_create(), pthread_setname_np() and
- * pthread_setaffinity_np(). The dataplane and service lcores are
- * excluded from the affinity of the new thread.
+ * pthread_setaffinity_np(). The affinity of the new thread is based
+ * on the cpu affinity retrieved at the time rte_eal_init() was called,
+ * the dataplane and service lcores are then excluded.
  *
  * @param thread
  *   Filled with the thread id of the new created thread.
Signature · 1 line
1.8.3.1
Reviewed-by on v3 2/2
#16 lore raw
Quoted 48 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity is lost.

Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1057
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1057 100131 testpmd             -                     2    1 2
  1057 100140 testpmd             eal-intr-thread       1    1 0-1
  1057 100141 testpmd             rte_mp_handle         1    1 0-1
  1057 100142 testpmd             lcore-slave-3         3    1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1061
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1061 100131 testpmd             -                     2    2 2
  1061 100144 testpmd             eal-intr-thread       1    2 1
  1061 100145 testpmd             rte_mp_handle         1    2 1
  1061 100147 testpmd             lcore-slave-3         3    2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1065
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1065 100131 testpmd             -                     2    2 2
  1065 100148 testpmd             eal-intr-thread       2    2 2
  1065 100149 testpmd             rte_mp_handle         2    2 2
  1065 100150 testpmd             lcore-slave-3         3    2 3
Quoted 4 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Signature · 2 lines
Thanks,
Anatoly
Reviewed-by
#17 lore raw
Quoted 49 lines · Anatoly
Quoted 48 lines · David Marchand

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity is lost.

Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1057
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1057 100131 testpmd             -                     2    1 2
  1057 100140 testpmd             eal-intr-thread       1    1 0-1
  1057 100141 testpmd             rte_mp_handle         1    1 0-1
  1057 100142 testpmd             lcore-slave-3         3    1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1061
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1061 100131 testpmd             -                     2    2 2
  1061 100144 testpmd             eal-intr-thread       1    2 1
  1061 100145 testpmd             rte_mp_handle         1    2 1
  1061 100147 testpmd             lcore-slave-3         3    2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
  -- -i --total-num-mbufs=2048
$ procstat -S 1065
   PID    TID COMM                TDNAME              CPU CSID CPU MASK
  1065 100131 testpmd             -                     2    2 2
  1065 100148 testpmd             eal-intr-thread       2    2 2
  1065 100149 testpmd             rte_mp_handle         2    2 2
  1065 100150 testpmd             lcore-slave-3         3    2 3
Quoted 6 lines
Quoted 4 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")

---

#18 lore raw

25/02/2019 09:33, Olivier Matz:

Quoted 15 lines · Anatoly
Quoted 14 lines · David Marchand
Quoted 13 lines

Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity is lost.

[...]

Quoted 9 lines
Quoted 7 lines
Quoted 5 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Replaced cpu by uppercase CPU,
and applied, thanks

Reviewed-by on v3 1/2
#19 lore raw

Re: [PATCH v3 1/2] eal: fix potential incorrect pinning for ctrl threads

In reply to David Marchand
Quoted 8 lines · David Marchand

pthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")

---

Signature · 2 lines
Thanks,
Anatoly
Reviewed-by
#20 lore raw

Re: [PATCH v3 1/2] eal: fix potential incorrect pinning for ctrl threads

Quoted 11 lines · Anatoly
Quoted 8 lines · David Marchand

pthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.

Fixes d651ee4919cd ("eal: set affinity for control threads")

---