eal: restrict ctrl threads to startup cpu affinity
PATCH 1/1 eal: restrict ctrl threads to startup cpu affinity
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.
Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.
The cpuset is computed once at init before the original cpu affinity.
--- lib/librte_eal/common/eal_common_options.c | 28 ++++++++++++++++++++++++++++ lib/librte_eal/common/eal_common_thread.c | 21 ++++----------------- lib/librte_eal/common/eal_internal_cfg.h | 3 +++ 3 files changed, 35 insertions(+), 17 deletions(-)
lib/librte_eal/common/eal_common_options.c +28 -0
@@ -217,6 +217,7 @@ struct device_option { internal_cfg->create_uio_dev =0 ; internal_cfg->iova_mode =RTE_IOVA_DC ; internal_cfg->user_mbuf_pool_ops_name =NULL ; +CPU_ZERO (&internal_cfg->ctrl_cpuset); internal_cfg->init_complete =0 ; }
@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id; + +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) { +CPU_SET (lcore_id, cpuset); + } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set) <0 ) +CPU_ZERO (&default_set); + +CPU_AND (cpuset, cpuset, &default_set); + +/* if no detected cpu is off, use master core */ +if (!CPU_COUNT (cpuset)) +CPU_SET (rte_get_master_lcore (), cpuset); +} +int eal_cleanup_config (struct internal_config *internal_cfg) {
@@ -1393,6 +1419,8 @@ static int xdigit2val(unsigned char c) lcore_config[cfg->master_lcore].core_role =ROLE_RTE ; } +compute_ctrl_threads_cpuset (internal_cfg); +/* if no memory amounts were requested, this will result in 0 and * will be overridden later, right after eal_hugepage_info_init() */ for (i =0 ; i <RTE_MAX_NUMA_NODES ; i++)
lib/librte_eal/common/eal_common_thread.c +4 -17
@@ -16,6 +16,7 @@#include <rte_memory.h> #include <rte_log.h> +#include "eal_internal_cfg.h" #include "eal_private.h" #include "eal_thread.h"
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { + rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;struct rte_thread_ctrl_params *params; -unsigned int lcore_id; -rte_cpuset_t cpuset; -int cpu_found, ret; +int ret; params =malloc (sizeof (*params));if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg) "Cannot set name for ctrl thread\n"); } - cpu_found =0 ; -CPU_ZERO (&cpuset); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { -if (eal_cpu_detected (lcore_id) && -rte_lcore_has_role (lcore_id,ROLE_OFF )) { -CPU_SET (lcore_id, &cpuset); - cpu_found =1 ; - } - } -/* if no detected cpu is off, use master core */ -if (!cpu_found) -CPU_SET (rte_get_master_lcore (), &cpuset); - - ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); + ret =pthread_setaffinity_np (*thread,sizeof (*cpuset), cpuset);if (ret <0 ) goto fail;
lib/librte_eal/common/eal_internal_cfg.h +3 -0
@@ -13,6 +13,8 @@#include <rte_eal.h> #include <rte_pci_dev_feature_defs.h> +#include "eal_thread.h" +#define MAX_HUGEPAGE_SIZES 3/**< support up to 3 page sizes */ /*
@@ -73,6 +75,7 @@ struct internal_config {unsigned num_hugepage_sizes;/**< how many sizes on this system */ struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES ];enum rte_iova_mode iova_mode ;/**< Set IOVA mode on this system */ +rte_cpuset_t ctrl_cpuset;/**< cpuset for ctrl threads */ volatile unsigned int init_complete;/**< indicates whether EAL has completed initialization */ };
Signature · 1 line
1.8.3.1
Quoted 10 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.
Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.The cpuset is computed once at init before the original cpu affinity.
Need to fix this last sentence...
Quoted 28 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>--- lib/librte_eal/common/eal_common_options.c | 28 ++++++++++++++++++++++++++++ lib/librte_eal/common/eal_common_thread.c | 21 ++++----------------- lib/librte_eal/common/eal_internal_cfg.h | 3 +++ 3 files changed, 35 insertions(+), 17 deletions(-)lib/librte_eal/common/eal_common_options.c +10
-0@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id; + +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) {
lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1383
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +9
-0+CPU_SET (lcore_id, cpuset); + } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set) <0 ) +CPU_ZERO (&default_set); + +CPU_AND (cpuset, cpuset, &default_set);
CPU_AND is different on Freebsd.
*CPU*_*AND*(*cpuset*_*t* **dst*, *cpuset*_*t* **src*);
The *CPU*_*AND*() macro removes CPUs absent from *src* from
*dst*. (It is the
*cpuset(9)*
<https://www.freebsd.org/cgi/man.cgi?query=cpuset&sektion=9&apropos=0&manpath=FreeBSD+11.0-RELEASE+and+Ports>
equivalent of the scalar: *dst* &= *src*.) *CPU*_*AND*_*ATOMIC*() is
similar, with the same atomic semantics as *CPU*_*OR*_*ATOMIC*().
Will fix in v2.
Signature · 1 line
David Marchand
Quoted 8 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.
Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.
Quoted 5 lines
The cpuset is computed once at init before the original cpu affinity.
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Hi David,
Maybe i didn't have enough coffee today and i'm missing something here, but how is this different? Removing the coremask cores from the cpuset will effectively "spawn the ctrl threads on anything that is not part of the EAL coremask" (which is "not that polite to the rest of the system"), unless the application was run with taskset.
Is "taskset" the key point here? I.e. by default, we're still "not polite", unless the user asks nicely? :)
Signature · 2 lines
Thanks, Anatoly
Quoted 17 lines · Anatoly
Quoted 8 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.
Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.Quoted 5 lines
The cpuset is computed once at init before the original cpu affinity.
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Hi David,
Quoted 9 lines
Maybe i didn't have enough coffee today and i'm missing something here, but how is this different? Removing the coremask cores from the cpuset will effectively "spawn the ctrl threads on anything that is not part of the EAL coremask" (which is "not that polite to the rest of the system"), unless the application was run with taskset.
Is "taskset" the key point here? I.e. by default, we're still "not polite", unless the user asks nicely? :)
Eheh, sorry, yes.
A bit more context then, if you want to clearly pin cpu resources for the processes on your system (let's say having virtual machines and a popular vswitch), I can only think of two solutions.
Either you have something to configure your processes to have them call sched_setaffinity/pthread_set_affinity_np, or you use taskset to get them "jailed" without them caring.
Before the incriminated commit, we were keeping all threads on the coremask that had been passed, but as Olivier said, we would end up with ctrl threads spanwed on core running dataplane threads as well.
Now, the ctrl threads can be spawned anywhere on all & ~coremask, with no way to configure this.
I considered adding a new eal option, but I think relying on the current cpu affinity is a better default behavior and I can't see drawbacks at the moment.
Signature · 1 line
David Marchand
Quoted 31 lines · David Marchand
On Thu, Feb 14, 2019 at 10:39 AM Burakov, Anatoly <[email protected] <mailto:[email protected]>> wrote:
On 13-Feb-19 4:13 PM, David Marchand wrote:
> Spawning the ctrl threads on anything that is not part of the eal > coremask is not that polite to the rest of the system.
>
> Rather than introduce yet another eal options for this, let's take > the startup cpu affinity as a reference and remove the eal coremask > from it.
> If no cpu is left, then we default to the master core.
>
> The cpuset is computed once at init before the original cpu affinity. >
> Fixes:d651ee4919cd("eal: set affinity for control threads") > Signed-off-by: David Marchand <[email protected]
<mailto:[email protected]>>
> ---Hi David,
Maybe i didn't have enough coffee today and i'm missing something here, but how is this different? Removing the coremask cores from the cpuset will effectively "spawn the ctrl threads on anything that is not part of
the EAL coremask" (which is "not that polite to the rest of the system"), unless the application was run with taskset.Is "taskset" the key point here? I.e. by default, we're still "not polite", unless the user asks nicely? :)
Quoted 13 lines
Eheh, sorry, yes.
A bit more context then, if you want to clearly pin cpu resources for the processes on your system (let's say having virtual machines and a popular vswitch), I can only think of two solutions. Either you have something to configure your processes to have them call sched_setaffinity/pthread_set_affinity_np, or you use taskset to get them "jailed" without them caring.Before the incriminated commit, we were keeping all threads on the coremask that had been passed, but as Olivier said, we would end up with ctrl threads spanwed on core running dataplane threads as well.
Now, the ctrl threads can be spawned anywhere on all & ~coremask, with no way to configure this.
I considered adding a new eal option, but I think relying on the current cpu affinity is a better default behavior and I can't see drawbacks at the moment.Signature · 1 line
David Marchand
OK, that makes sense. However, i feel this behavior (both old and new, for that matter) should be better documented somewhere in the EAL docs.
Signature · 2 lines
Thanks, Anatoly
Quoted 13 lines · Anatoly
Quoted 11 lines · David Marchand
A bit more context then, if you want to clearly pin cpu resources for the processes on your system (let's say having virtual machines and a popular vswitch), I can only think of two solutions.
Either you have something to configure your processes to have them call sched_setaffinity/pthread_set_affinity_np, or you use taskset to get them "jailed" without them caring.Before the incriminated commit, we were keeping all threads on the coremask that had been passed, but as Olivier said, we would end up with ctrl threads spanwed on core running dataplane threads as well.
Quoted 10 lines
Quoted 5 lines
Now, the ctrl threads can be spawned anywhere on all & ~coremask, with no way to configure this.
I considered adding a new eal option, but I think relying on the current cpu affinity is a better default behavior and I can't see drawbacks at the moment.OK, that makes sense. However, i feel this behavior (both old and new, for that matter) should be better documented somewhere in the EAL docs.
I'll see what I can add in the doc for v2.
Signature · 1 line
David Marchand
Quoted 36 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system.
Rather than introduce yet another eal options for this, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.The cpuset is computed once at init before the original cpu affinity.
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>--- lib/librte_eal/common/eal_common_options.c | 28 ++++++++++++++++++++++++++++ lib/librte_eal/common/eal_common_thread.c | 21 ++++----------------- lib/librte_eal/common/eal_internal_cfg.h | 3 +++ 3 files changed, 35 insertions(+), 17 deletions(-)lib/librte_eal/common/eal_common_options.c +8
-0@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id; + +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {
lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1381
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +9
-0+if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) { +CPU_SET (lcore_id, cpuset); + } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set) <0 ) +CPU_ZERO (&default_set);
Now that I am testing on Freebsd, I can see this block I took from the existing "auto detect" function is just wrong. pthread_(g|s)etaffinity_np return a > 0 error value when failing.
Signature · 1 line
David Marchand
PATCH v2 1/2 eal: fix potential incorrect pinning for ctrl threads
In reply to David Marchandpthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.
--- lib/librte_eal/common/eal_common_thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/librte_eal/common/eal_common_thread.c +1 -1
@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)CPU_SET (rte_get_master_lcore (), &cpuset); ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); -if (ret <0 ) +if (ret) goto fail; ret =pthread_barrier_wait (¶ms->configured);
Signature · 1 line
1.8.3.1
PATCH v2 2/2 eal: restrict ctrl threads to startup cpu affinity
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).
Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.
The cpuset is computed once at init before the original cpu affinity is lost.
Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.
Examples in a 4 cores FreeBSD vm:
$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048
$ procstat -S 1057 PID TID COMM TDNAME CPU CSID CPU MASK 1057 100131 testpmd - 2 1 2 1057 100140 testpmd eal-intr-thread 1 1 0-1 1057 100141 testpmd rte_mp_handle 1 1 0-1 1057 100142 testpmd lcore-slave-3 3 1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048
$ procstat -S 1061 PID TID COMM TDNAME CPU CSID CPU MASK 1061 100131 testpmd - 2 2 2 1061 100144 testpmd eal-intr-thread 1 2 1 1061 100145 testpmd rte_mp_handle 1 2 1 1061 100147 testpmd lcore-slave-3 3 2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048
$ procstat -S 1065 PID TID COMM TDNAME CPU CSID CPU MASK 1065 100131 testpmd - 2 2 2 1065 100148 testpmd eal-intr-thread 2 2 2 1065 100149 testpmd rte_mp_handle 2 2 2 1065 100150 testpmd lcore-slave-3 3 2 3
---
Changes since v1:
- added some description in the prog guide
- fixed FreeBSD build
--- doc/guides/prog_guide/env_abstraction_layer.rst | 14 +++++++++++++ lib/librte_eal/common/eal_common_options.c | 28 +++++++++++++++++++++++++ lib/librte_eal/common/eal_common_thread.c | 21 ++++--------------- lib/librte_eal/common/eal_internal_cfg.h | 3 +++ lib/librte_eal/common/include/rte_lcore.h | 17 +++++++++++---- 5 files changed, 62 insertions(+), 21 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +14 -0
@@ -498,6 +498,20 @@ Those TLS include *_cpuset* and *_socket_id*: * *_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY. +Control Thread API +~~~~~~~~~~~~~~~~~~ + +It is possible to create Control Threads using the public API ``rte_ctrl_thread_create()``. +Those threads can be used for management/infrastructure tasks and are used internally by DPDK for multi process support and interrupt handling. + +Those threads will be scheduled on cpus part of the original process cpu affinity from which the dataplane and service lcores are excluded. + +For example, on a 8 cpus system, starting a dpdk application with -l 2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD), + +- with no affinity configuration, the Control Threads will end up on 0-1,4-7 cpus. +- with affinity restricted to 2-4, the Control Threads will end up on cpu 4. +- with affinity restricted to 2-3, the Control Threads will end up on cpu 2 (master lcore, which is the default when no cpu is available). + .. _known_issue_label: Known Issues
lib/librte_eal/common/eal_common_options.c +28 -0
@@ -217,6 +217,7 @@ struct device_option { internal_cfg->create_uio_dev =0 ; internal_cfg->iova_mode =RTE_IOVA_DC ; internal_cfg->user_mbuf_pool_ops_name =NULL ; +CPU_ZERO (&internal_cfg->ctrl_cpuset); internal_cfg->init_complete =0 ; }
@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id; + +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) { +CPU_SET (lcore_id, cpuset); + } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set) <0 ) +CPU_ZERO (&default_set); + +RTE_CPU_AND (cpuset, cpuset, &default_set); + +/* if no detected cpu is off, use master core */ +if (!CPU_COUNT (cpuset)) +CPU_SET (rte_get_master_lcore (), cpuset); +} +int eal_cleanup_config (struct internal_config *internal_cfg) {
@@ -1392,6 +1418,8 @@ static int xdigit2val(unsigned char c) lcore_config[cfg->master_lcore].core_role =ROLE_RTE ; } +compute_ctrl_threads_cpuset (internal_cfg); +/* if no memory amounts were requested, this will result in 0 and * will be overridden later, right after eal_hugepage_info_init() */ for (i =0 ; i <RTE_MAX_NUMA_NODES ; i++)
lib/librte_eal/common/eal_common_thread.c +4 -17
@@ -16,6 +16,7 @@#include <rte_memory.h> #include <rte_log.h> +#include "eal_internal_cfg.h" #include "eal_private.h" #include "eal_thread.h"
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { + rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;struct rte_thread_ctrl_params *params; -unsigned int lcore_id; -rte_cpuset_t cpuset; -int cpu_found, ret; +int ret; params =malloc (sizeof (*params));if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg) "Cannot set name for ctrl thread\n"); } - cpu_found =0 ; -CPU_ZERO (&cpuset); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { -if (eal_cpu_detected (lcore_id) && -rte_lcore_has_role (lcore_id,ROLE_OFF )) { -CPU_SET (lcore_id, &cpuset); - cpu_found =1 ; - } - } -/* if no detected cpu is off, use master core */ -if (!cpu_found) -CPU_SET (rte_get_master_lcore (), &cpuset); - - ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); + ret =pthread_setaffinity_np (*thread,sizeof (*cpuset), cpuset);if (ret) goto fail;
lib/librte_eal/common/eal_internal_cfg.h +3 -0
@@ -13,6 +13,8 @@#include <rte_eal.h> #include <rte_pci_dev_feature_defs.h> +#include "eal_thread.h" +#define MAX_HUGEPAGE_SIZES 3/**< support up to 3 page sizes */ /*
@@ -73,6 +75,7 @@ struct internal_config {unsigned num_hugepage_sizes;/**< how many sizes on this system */ struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES ];enum rte_iova_mode iova_mode ;/**< Set IOVA mode on this system */ +rte_cpuset_t ctrl_cpuset;/**< cpuset for ctrl threads */ volatile unsigned int init_complete;/**< indicates whether EAL has completed initialization */ };
lib/librte_eal/common/include/rte_lcore.h +13 -4
@@ -23,10 +23,18 @@#define LCORE_ID_ANY UINT32_MAX/**< Any lcore. */ #if defined(__linux__) -typedef cpu_set_t rte_cpuset_t ; +typedef cpu_set_t rte_cpuset_t ; +#define RTE_CPU_AND (dst, src1, src2) CPU_AND(dst, src1, src2)#elif defined(__FreeBSD__)#include <pthread_np.h> -typedef cpuset_t rte_cpuset_t ; +typedef cpuset_t rte_cpuset_t ; +#define RTE_CPU_AND (dst, src1, src2) do \ +{ \ + cpuset_t tmp; \ + CPU_COPY(src1, &tmp); \ + CPU_AND(&tmp, src2); \ + CPU_COPY(&tmp, dst); \ +} while (0)#endif /**
@@ -280,8 +288,9 @@ struct lcore_config {* Create a control thread. * * Wrapper to pthread_create(), pthread_setname_np() and -* pthread_setaffinity_np(). The dataplane and service lcores are -* excluded from the affinity of the new thread. +* pthread_setaffinity_np(). The affinity of the new thread is based +* on the cpu affinity retrieved at the time rte_eal_init() was called, +* the dataplane and service lcores are then excluded. * * @param thread * Filled with the thread id of the new created thread.
Signature · 1 line
1.8.3.1
Quoted 75 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).
Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.The cpuset is computed once at init before the original cpu affinity is lost.
Introduced a
RTE_CPU_ANDmacro to abstract the differences between linux and freebsd respective macros.Examples in a 4 cores FreeBSD vm:
$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1057 PID TID COMM TDNAME CPU CSID CPU MASK 1057 100131 testpmd - 2 1 2 1057 100140 testpmd eal-intr-thread 1 1 0-1 1057 100141 testpmd rte_mp_handle 1 1 0-1 1057 100142 testpmd lcore-slave-3 3 1 3$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1061 PID TID COMM TDNAME CPU CSID CPU MASK 1061 100131 testpmd - 2 2 2 1061 100144 testpmd eal-intr-thread 1 2 1 1061 100145 testpmd rte_mp_handle 1 2 1 1061 100147 testpmd lcore-slave-3 3 2 3$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1065 PID TID COMM TDNAME CPU CSID CPU MASK 1065 100131 testpmd - 2 2 2 1065 100148 testpmd eal-intr-thread 2 2 2 1065 100149 testpmd rte_mp_handle 2 2 2 1065 100150 testpmd lcore-slave-3 3 2 3Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Changes since v1:
- added some description in the prog guide
- fixed FreeBSD build--- doc/guides/prog_guide/env_abstraction_layer.rst | 14 +++++++++++++ lib/librte_eal/common/eal_common_options.c | 28 +++++++++++++++++++++++++ lib/librte_eal/common/eal_common_thread.c | 21 ++++--------------- lib/librte_eal/common/eal_internal_cfg.h | 3 +++ lib/librte_eal/common/include/rte_lcore.h | 17 +++++++++++---- 5 files changed, 62 insertions(+), 21 deletions(-)doc/guides/prog_guide/env_abstraction_layer.rst +3
-0@@ -498,6 +498,20 @@ Those TLS include *_cpuset* and *_socket_id*: * *_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY. +Control Thread API +~~~~~~~~~~~~~~~~~~ +
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 513
Quoted 10 lines
doc/guides/prog_guide/env_abstraction_layer.rst +10
-0+It is possible to create Control Threads using the public API ``rte_ctrl_thread_create()``. +Those threads can be used for management/infrastructure tasks and are used internally by DPDK for multi process support and interrupt handling. + +Those threads will be scheduled on cpus part of the original process cpu affinity from which the dataplane and service lcores are excluded. + +For example, on a 8 cpus system, starting a dpdk application with -l 2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD), + +- with no affinity configuration, the Control Threads will end up on 0-1,4-7 cpus. +- with affinity restricted to 2-4, the Control Threads will end up on cpu 4. +- with affinity restricted to 2-3, the Control Threads will end up on cpu 2 (master lcore, which is the default when no cpu is available).
You're not winning anything by foregoing the 80 char limit on documentation (doxygen will still generate this correctly), but you're losing in readability when working in terminal. I would prefer if you didn't do those long lines :)
Thomas, do we want checkpatch to warn about this?
Quoted 26 lines
doc/guides/prog_guide/env_abstraction_layer.rst +1
-0+ .. _known_issue_label: Known Issueslib/librte_eal/common/eal_common_options.c +7
-0@@ -217,6 +217,7 @@ struct device_option { internal_cfg->create_uio_dev =0 ; internal_cfg->iova_mode =RTE_IOVA_DC ; internal_cfg->user_mbuf_pool_ops_name =NULL ; +CPU_ZERO (&internal_cfg->ctrl_cpuset); internal_cfg->init_complete =0 ; }@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id;
lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1378
Quoted 10 lines
lib/librte_eal/common/eal_common_options.c +10
-0+ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) { +CPU_SET (lcore_id, cpuset); + } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set) <0 )
Shouldn't this be != 0? Manpage doesn't say the error values will be negative.
Signature · 2 lines
Thanks, Anatoly
Quoted 27 lines · Anatoly
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 498
Quoted 4 lines · David Marchand
doc/guides/prog_guide/env_abstraction_layer.rst +0
-0@@ -498,6 +498,20 @@ Those TLS include *_cpuset* and *_socket_id*: * *_socket_id* stores the NUMA node of the CPU set. If the CPUs inCPU set belong to different NUMA node, the *_socket_id* will be set to
SOCKET_ID_ANY.doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*:
Quoted 6 lines
doc/guides/prog_guide/env_abstraction_layer.rst +4
-0+Control Thread API +~~~~~~~~~~~~~~~~~~ + +It is possible to create Control Threads using the public API``rte_ctrl_thread_create()``.
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 503
Quoted 1 line
doc/guides/prog_guide/env_abstraction_layer.rst +1
-0+Those threads can be used for management/infrastructure tasks and areused internally by DPDK for multi process support and interrupt handling.
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 505
Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2
-0+ +Those threads will be scheduled on cpus part of the original processcpu affinity from which the dataplane and service lcores are excluded.
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 507
Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2
-0+ +For example, on a 8 cpus system, starting a dpdk application with -l2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD),
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 509
Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2
-0+ +- with no affinity configuration, the Control Threads will end up on0-1,4-7 cpus.
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 510
Quoted 1 line
doc/guides/prog_guide/env_abstraction_layer.rst +1
-0+- with affinity restricted to 2-4, the Control Threads will end up on
Quoted 9 lines
cpu 4.
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*: · line 511
Quoted 1 line
doc/guides/prog_guide/env_abstraction_layer.rst +1
-0+- with affinity restricted to 2-3, the Control Threads will end up oncpu 2 (master lcore, which is the default when no cpu is available).
You're not winning anything by foregoing the 80 char limit on documentation (doxygen will still generate this correctly), but you're losing in readability when working in terminal. I would prefer if you didn't do those long lines :)
I don't really care, I will just wait for Thomas opinion.
Quoted 34 lines
Thomas, do we want checkpatch to warn about this?
Review
Quoted 5 lines
doc/guides/prog_guide/env_abstraction_layer.rst +1
-0+ .. _known_issue_label: Known Issues(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_options.cQuoted 26 lines
index 1f45f82..fca3f83 100644
lib/librte_eal/common/eal_common_options.c +12
-0@@ -217,6 +217,7 @@ struct device_option { internal_cfg->create_uio_dev =0 ; internal_cfg->iova_mode =RTE_IOVA_DC ; internal_cfg->user_mbuf_pool_ops_name =NULL ; +CPU_ZERO (&internal_cfg->ctrl_cpuset); internal_cfg->init_complete =0 ; }@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id; + +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) { +CPU_SET (lcore_id, cpuset);
Quoted 9 lines
lib/librte_eal/common/eal_common_options.c static int xdigit2val(unsigned char c) · line 1378
Quoted 5 lines
lib/librte_eal/common/eal_common_options.c +5
-0+ } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set) <0 )Shouldn't this be != 0? Manpage doesn't say the error values will be negative.
Good catch...
/me hides
Thanks for the review.
Signature · 1 line
David Marchand
19/02/2019 12:51, David Marchand:
Quoted 25 lines
Quoted 23 lines · Anatoly
doc/guides/prog_guide/env_abstraction_layer.rst Those TLS include *_cpuset* and *_socket_id*:
Quoted 6 lines · David Marchand
doc/guides/prog_guide/env_abstraction_layer.rst +4
-0+Control Thread API +~~~~~~~~~~~~~~~~~~ + +It is possible to create Control Threads using the public API``rte_ctrl_thread_create()``.
Quoted 1 line
+Those threads can be used for management/infrastructure tasks and are
used internally by DPDK for multi process support and interrupt handling.
doc/guides/prog_guide/env_abstraction_layer.rst
Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2
-0+ +Those threads will be scheduled on cpus part of the original processcpu affinity from which the dataplane and service lcores are excluded.
doc/guides/prog_guide/env_abstraction_layer.rst
Quoted 2 lines
doc/guides/prog_guide/env_abstraction_layer.rst +2
-0+ +For example, on a 8 cpus system, starting a dpdk application with -l2,3 (dataplane cores), then depending on the affinity configuration which can be controlled with tools like taskset (Linux) or cpuset (FreeBSD),
Quoted 2 lines
+
+- with no affinity configuration, the Control Threads will end up on0-1,4-7 cpus.
Quoted 1 line
+- with affinity restricted to 2-4, the Control Threads will end up on
cpu 4.
Quoted 1 line
+- with affinity restricted to 2-3, the Control Threads will end up on
Quoted 10 lines
Quoted 6 lines
cpu 2 (master lcore, which is the default when no cpu is available).
You're not winning anything by foregoing the 80 char limit on documentation (doxygen will still generate this correctly), but you're losing in readability when working in terminal. I would prefer if you didn't do those long lines :)
I don't really care, I will just wait for Thomas opinion.
Quoted 1 line
Thomas, do we want checkpatch to warn about this?
http://doc.dpdk.org/guides/contributing/documentation.html#line-length
Lines should not exceed 80 chars.
There is a tradition of being very flexible in docs.
The best is to wrap at the end of a logical group of words, like after commas or dots. So the doc updates patches are easier to read.
Re: [PATCH v2 1/2] eal: fix potential incorrect pinning for ctrl threads
In reply to David MarchandQuoted 16 lines · David Marchand
pthread_setaffinity_npreturns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>--- lib/librte_eal/common/eal_common_thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)lib/librte_eal/common/eal_common_thread.c +0
-0
lib/librte_eal/common/eal_common_thread.c static void *rte_thread_init(void *arg) · line 215
Quoted 8 lines
lib/librte_eal/common/eal_common_thread.c +1
-1@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)CPU_SET (rte_get_master_lcore (), &cpuset); ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); -if (ret <0 ) +if (ret) goto fail; ret =pthread_barrier_wait (¶ms->configured);
Signature · 2 lines
Thanks, Anatoly
Re: [PATCH v2 1/2] eal: fix potential incorrect pinning for ctrl threads
Quoted 24 lines · Anatoly
Review
Quoted 11 lines · David Marchand
pthread_setaffinity_npreturns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>--- lib/librte_eal/common/eal_common_thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)(quoted diff) +0
-0
b/lib/librte_eal/common/eal_common_thread.cQuoted 11 lines
index 48ef4d6..a3985ce 100644
lib/librte_eal/common/eal_common_thread.c +1
-1@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)CPU_SET (rte_get_master_lcore (), &cpuset); ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); -if (ret <0 ) +if (ret) goto fail;
Quoted 5 lines
lib/librte_eal/common/eal_common_thread.c static void *rte_thread_init(void *arg) · line 215
Quoted 2 lines
lib/librte_eal/common/eal_common_thread.c +0
-0ret =pthread_barrier_wait (¶ms->configured);CC stable?
Yes, I again forgot to check with check-git-log.sh...
Signature · 1 line
David Marchand
PATCH v3 1/2 eal: fix potential incorrect pinning for ctrl threads
In reply to David Marchandpthread_setaffinity_np returns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.
---
Changelog since v2:
- added missing Cc: [email protected]
--- lib/librte_eal/common/eal_common_thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/librte_eal/common/eal_common_thread.c +1 -1
@@ -209,7 +209,7 @@ static void *rte_thread_init(void *arg)CPU_SET (rte_get_master_lcore (), &cpuset); ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); -if (ret <0 ) +if (ret) goto fail; ret =pthread_barrier_wait (¶ms->configured);
Signature · 1 line
1.8.3.1
PATCH v3 2/2 eal: restrict ctrl threads to startup cpu affinity
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).
Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.
The cpuset is computed once at init before the original cpu affinity is lost.
Introduced a RTE_CPU_AND macro to abstract the differences between linux and freebsd respective macros.
Examples in a 4 cores FreeBSD vm:
$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048
$ procstat -S 1057 PID TID COMM TDNAME CPU CSID CPU MASK 1057 100131 testpmd - 2 1 2 1057 100140 testpmd eal-intr-thread 1 1 0-1 1057 100141 testpmd rte_mp_handle 1 1 0-1 1057 100142 testpmd lcore-slave-3 3 1 3
$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048
$ procstat -S 1061 PID TID COMM TDNAME CPU CSID CPU MASK 1061 100131 testpmd - 2 2 2 1061 100144 testpmd eal-intr-thread 1 2 1 1061 100145 testpmd rte_mp_handle 1 2 1 1061 100147 testpmd lcore-slave-3 3 2 3
$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048
$ procstat -S 1065 PID TID COMM TDNAME CPU CSID CPU MASK 1065 100131 testpmd - 2 2 2 1065 100148 testpmd eal-intr-thread 2 2 2 1065 100149 testpmd rte_mp_handle 2 2 2 1065 100150 testpmd lcore-slave-3 3 2 3
---
Changes since v2:
- added missing Cc: [email protected]
- re-indent doc to comply with 80 columns
- fix incorrect check on pthread_getaffinity_np
Changes since v1:
- added some description in the prog guide
- fixed FreeBSD build
--- doc/guides/prog_guide/env_abstraction_layer.rst | 22 +++++++++++++++++++ lib/librte_eal/common/eal_common_options.c | 28 +++++++++++++++++++++++++ lib/librte_eal/common/eal_common_thread.c | 21 ++++--------------- lib/librte_eal/common/eal_internal_cfg.h | 3 +++ lib/librte_eal/common/include/rte_lcore.h | 17 +++++++++++---- 5 files changed, 70 insertions(+), 21 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +22 -0
@@ -498,6 +498,28 @@ Those TLS include *_cpuset* and *_socket_id*: * *_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY. +Control Thread API +~~~~~~~~~~~~~~~~~~ + +It is possible to create Control Threads using the public API +``rte_ctrl_thread_create()``. +Those threads can be used for management/infrastructure tasks and are used +internally by DPDK for multi process support and interrupt handling. + +Those threads will be scheduled on cpus part of the original process cpu +affinity from which the dataplane and service lcores are excluded. + +For example, on a 8 cpus system, starting a dpdk application with -l 2,3 +(dataplane cores), then depending on the affinity configuration which can be +controlled with tools like taskset (Linux) or cpuset (FreeBSD), + +- with no affinity configuration, the Control Threads will end up on + 0-1,4-7 cpus. +- with affinity restricted to 2-4, the Control Threads will end up on + cpu 4. +- with affinity restricted to 2-3, the Control Threads will end up on + cpu 2 (master lcore, which is the default when no cpu is available). + .. _known_issue_label: Known Issues
lib/librte_eal/common/eal_common_options.c +28 -0
@@ -217,6 +217,7 @@ struct device_option { internal_cfg->create_uio_dev =0 ; internal_cfg->iova_mode =RTE_IOVA_DC ; internal_cfg->user_mbuf_pool_ops_name =NULL ; +CPU_ZERO (&internal_cfg->ctrl_cpuset); internal_cfg->init_complete =0 ; }
@@ -1359,6 +1360,31 @@ static int xdigit2val(unsigned char c) cfg->lcore_count -= removed; } +static void +compute_ctrl_threads_cpuset (struct internal_config *internal_cfg) +{ +rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset; +rte_cpuset_t default_set; +unsigned int lcore_id; + +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +if (eal_cpu_detected (lcore_id) && +rte_lcore_has_role (lcore_id,ROLE_OFF )) { +CPU_SET (lcore_id, cpuset); + } + } + +if (pthread_getaffinity_np (pthread_self (),sizeof (rte_cpuset_t), + &default_set)) +CPU_ZERO (&default_set); + +RTE_CPU_AND (cpuset, cpuset, &default_set); + +/* if no detected cpu is off, use master core */ +if (!CPU_COUNT (cpuset)) +CPU_SET (rte_get_master_lcore (), cpuset); +} +int eal_cleanup_config (struct internal_config *internal_cfg) {
@@ -1392,6 +1418,8 @@ static int xdigit2val(unsigned char c) lcore_config[cfg->master_lcore].core_role =ROLE_RTE ; } +compute_ctrl_threads_cpuset (internal_cfg); +/* if no memory amounts were requested, this will result in 0 and * will be overridden later, right after eal_hugepage_info_init() */ for (i =0 ; i <RTE_MAX_NUMA_NODES ; i++)
lib/librte_eal/common/eal_common_thread.c +4 -17
@@ -16,6 +16,7 @@#include <rte_memory.h> #include <rte_log.h> +#include "eal_internal_cfg.h" #include "eal_private.h" #include "eal_thread.h"
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { + rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;struct rte_thread_ctrl_params *params; -unsigned int lcore_id; -rte_cpuset_t cpuset; -int cpu_found, ret; +int ret; params =malloc (sizeof (*params));if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg) "Cannot set name for ctrl thread\n"); } - cpu_found =0 ; -CPU_ZERO (&cpuset); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { -if (eal_cpu_detected (lcore_id) && -rte_lcore_has_role (lcore_id,ROLE_OFF )) { -CPU_SET (lcore_id, &cpuset); - cpu_found =1 ; - } - } -/* if no detected cpu is off, use master core */ -if (!cpu_found) -CPU_SET (rte_get_master_lcore (), &cpuset); - - ret =pthread_setaffinity_np (*thread,sizeof (cpuset), &cpuset); + ret =pthread_setaffinity_np (*thread,sizeof (*cpuset), cpuset);if (ret) goto fail;
lib/librte_eal/common/eal_internal_cfg.h +3 -0
@@ -13,6 +13,8 @@#include <rte_eal.h> #include <rte_pci_dev_feature_defs.h> +#include "eal_thread.h" +#define MAX_HUGEPAGE_SIZES 3/**< support up to 3 page sizes */ /*
@@ -73,6 +75,7 @@ struct internal_config {unsigned num_hugepage_sizes;/**< how many sizes on this system */ struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES ];enum rte_iova_mode iova_mode ;/**< Set IOVA mode on this system */ +rte_cpuset_t ctrl_cpuset;/**< cpuset for ctrl threads */ volatile unsigned int init_complete;/**< indicates whether EAL has completed initialization */ };
lib/librte_eal/common/include/rte_lcore.h +13 -4
@@ -23,10 +23,18 @@#define LCORE_ID_ANY UINT32_MAX/**< Any lcore. */ #if defined(__linux__) -typedef cpu_set_t rte_cpuset_t ; +typedef cpu_set_t rte_cpuset_t ; +#define RTE_CPU_AND (dst, src1, src2) CPU_AND(dst, src1, src2)#elif defined(__FreeBSD__)#include <pthread_np.h> -typedef cpuset_t rte_cpuset_t ; +typedef cpuset_t rte_cpuset_t ; +#define RTE_CPU_AND (dst, src1, src2) do \ +{ \ + cpuset_t tmp; \ + CPU_COPY(src1, &tmp); \ + CPU_AND(&tmp, src2); \ + CPU_COPY(&tmp, dst); \ +} while (0)#endif /**
@@ -280,8 +288,9 @@ struct lcore_config {* Create a control thread. * * Wrapper to pthread_create(), pthread_setname_np() and -* pthread_setaffinity_np(). The dataplane and service lcores are -* excluded from the affinity of the new thread. +* pthread_setaffinity_np(). The affinity of the new thread is based +* on the cpu affinity retrieved at the time rte_eal_init() was called, +* the dataplane and service lcores are then excluded. * * @param thread * Filled with the thread id of the new created thread.
Signature · 1 line
1.8.3.1
Quoted 48 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).
Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.The cpuset is computed once at init before the original cpu affinity is lost.
Introduced a
RTE_CPU_ANDmacro to abstract the differences between linux and freebsd respective macros.Examples in a 4 cores FreeBSD vm:
$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1057 PID TID COMM TDNAME CPU CSID CPU MASK 1057 100131 testpmd - 2 1 2 1057 100140 testpmd eal-intr-thread 1 1 0-1 1057 100141 testpmd rte_mp_handle 1 1 0-1 1057 100142 testpmd lcore-slave-3 3 1 3$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1061 PID TID COMM TDNAME CPU CSID CPU MASK 1061 100131 testpmd - 2 2 2 1061 100144 testpmd eal-intr-thread 1 2 1 1061 100145 testpmd rte_mp_handle 1 2 1 1061 100147 testpmd lcore-slave-3 3 2 3$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1065 PID TID COMM TDNAME CPU CSID CPU MASK 1065 100131 testpmd - 2 2 2 1065 100148 testpmd eal-intr-thread 2 2 2 1065 100149 testpmd rte_mp_handle 2 2 2 1065 100150 testpmd lcore-slave-3 3 2 3
Quoted 4 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Signature · 2 lines
Thanks, Anatoly
Olivier Matz
Reviewed-byQuoted 49 lines · Anatoly
Quoted 48 lines · David Marchand
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).
Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.The cpuset is computed once at init before the original cpu affinity is lost.
Introduced a
RTE_CPU_ANDmacro to abstract the differences between linux and freebsd respective macros.Examples in a 4 cores FreeBSD vm:
$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1057 PID TID COMM TDNAME CPU CSID CPU MASK 1057 100131 testpmd - 2 1 2 1057 100140 testpmd eal-intr-thread 1 1 0-1 1057 100141 testpmd rte_mp_handle 1 1 0-1 1057 100142 testpmd lcore-slave-3 3 1 3$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1061 PID TID COMM TDNAME CPU CSID CPU MASK 1061 100131 testpmd - 2 2 2 1061 100144 testpmd eal-intr-thread 1 2 1 1061 100145 testpmd rte_mp_handle 1 2 1 1061 100147 testpmd lcore-slave-3 3 2 3$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \ -- -i --total-num-mbufs=2048$ procstat -S 1065 PID TID COMM TDNAME CPU CSID CPU MASK 1065 100131 testpmd - 2 2 2 1065 100148 testpmd eal-intr-thread 2 2 2 1065 100149 testpmd rte_mp_handle 2 2 2 1065 100150 testpmd lcore-slave-3 3 2 3
Quoted 6 lines
Quoted 4 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Reviewed-by Anatoly Burakov <[email protected]>
25/02/2019 09:33, Olivier Matz:
Quoted 15 lines · Anatoly
Quoted 14 lines · David Marchand
Quoted 13 lines
Spawning the ctrl threads on anything that is not part of the eal coremask is not that polite to the rest of the system, especially when you took good care to pin your processes on cpu resources with tools like taskset (linux) / cpuset (freebsd).
Rather than introduce yet another eal options to control on which cpu those ctrl threads are created, let's take the startup cpu affinity as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.The cpuset is computed once at init before the original cpu affinity is lost.
[...]
Quoted 9 lines
Quoted 7 lines
Quoted 5 lines
Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Reviewed-by Anatoly Burakov <[email protected]>Reviewed-by Olivier Matz <[email protected]>
Replaced cpu by uppercase CPU,
and applied, thanks
Re: [PATCH v3 1/2] eal: fix potential incorrect pinning for ctrl threads
In reply to David MarchandQuoted 8 lines · David Marchand
pthread_setaffinity_npreturns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Signature · 2 lines
Thanks, Anatoly
Olivier Matz
Reviewed-byRe: [PATCH v3 1/2] eal: fix potential incorrect pinning for ctrl threads
Quoted 11 lines · Anatoly
Quoted 8 lines · David Marchand
pthread_setaffinity_npreturns a >0 value on error.
We could end up letting the ctrl threads on the current process cpu affinity.Fixes d651ee4919cd ("eal: set affinity for control threads")Signed-off-by David Marchand <[email protected]>---
Reviewed-by Anatoly Burakov <[email protected]>